From 34f2f3441ee4302893b39f5ce119742480d03f54 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 2 Jan 2025 10:23:48 +0200 Subject: [PATCH 01/41] Add codecs for lucene 8x --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 14 +- .../bwc/codecs/lucene70/Lucene70Codec.java | 1 + .../bwc/codecs/lucene86/BWCLucene86Codec.java | 153 ++++++++++++++++++ .../bwc/codecs/lucene87/BWCLucene87Codec.java | 153 ++++++++++++++++++ .../services/org.apache.lucene.codecs.Codec | 2 + 5 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 3ed8fc26ac937..b84dac328bf52 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; +import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec; +import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldsConsumer; @@ -26,6 +28,8 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; import java.io.IOException; import java.util.ArrayList; @@ -118,7 +122,15 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { } public static SegmentInfo wrap(SegmentInfo segmentInfo) { - final Codec codec = segmentInfo.getCodec(); + // special handling for Lucene87Codec (which is currently bundled with Lucene) + // Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs) + Codec codec = segmentInfo.getCodec(); + if (codec instanceof Lucene86Codec) { + codec = new BWCLucene86Codec(); + } else if (codec instanceof Lucene87Codec) { + codec = new BWCLucene87Codec(); + } + final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java index f9ba02676c2d0..141fbd3447c85 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java @@ -21,3 +21,4 @@ public Lucene70Codec() { super("Lucene70"); } } + diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java new file mode 100644 index 0000000000000..4372e44f8171c --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +import java.util.Objects; + +public class BWCLucene86Codec extends BWCCodec { + + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86PointsFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene86Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene86Codec.this.getDocValuesFormatForField(field); + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + /** Instantiates a new codec. */ + public BWCLucene86Codec() { + super("BWCLucene86Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(Lucene50StoredFieldsFormat.Mode.BEST_SPEED)); + this.defaultFormat = new Lucene84PostingsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return KnnVectorsFormat.EMPTY; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } + +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java new file mode 100644 index 0000000000000..7d36811a30626 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +public class BWCLucene87Codec extends BWCCodec { + + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86PointsFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene87Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene87Codec.this.getDocValuesFormatForField(field); + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + /** Instantiates a new codec. */ + public BWCLucene87Codec() { + super("BWCLucene87Codec"); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); + this.defaultFormat = new Lucene84PostingsFormat(); + this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return KnnVectorsFormat.EMPTY; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final DocValuesFormat defaultDVFormat; + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } + +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 0215e9f7ca4ab..7830dbffe4adf 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,6 +5,8 @@ # 2.0. # +org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec From 69b6b6e414d1fb73fd4b975184b23d965e843d07 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 2 Jan 2025 17:15:00 +0200 Subject: [PATCH 02/41] Add Lucene80 Codec support --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 6 +- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 123 ++++++++++++++++++ .../services/org.apache.lucene.codecs.Codec | 1 + 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index b84dac328bf52..37ca6f39068c0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; +import org.apache.lucene.backward_codecs.lucene80.Lucene80Codec; import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec; import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.Codec; @@ -28,6 +29,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; @@ -125,7 +127,9 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { // special handling for Lucene87Codec (which is currently bundled with Lucene) // Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs) Codec codec = segmentInfo.getCodec(); - if (codec instanceof Lucene86Codec) { + if (codec instanceof Lucene80Codec) { + codec = new BWCLucene80Codec(); + } else if (codec instanceof Lucene86Codec) { codec = new BWCLucene86Codec(); } else if (codec instanceof Lucene87Codec) { codec = new BWCLucene87Codec(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java new file mode 100644 index 0000000000000..5b83b3984cd5a --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + + +public class BWCLucene80Codec extends BWCCodec { + + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + + private final PostingsFormat postingsFormat = + new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + throw new UnsupportedOperationException("Old codecs can't be used for writing"); + } + }; + + private final DocValuesFormat docValuesFormat = + new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); + + private final StoredFieldsFormat storedFieldsFormat; + + /** Instantiates a new codec. */ + public BWCLucene80Codec() { + super("BWCLucene80Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public final PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public final SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public final CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public final PointsFormat pointsFormat() { + return new Lucene60PointsFormat(); + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public final NormsFormat normsFormat() { + return normsFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return KnnVectorsFormat.EMPTY; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 7830dbffe4adf..771fc7a0f19b3 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -7,6 +7,7 @@ org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec From 30e2cb4159c273efb7467cc27e08d574e64d59b1 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 2 Jan 2025 17:47:25 +0200 Subject: [PATCH 03/41] Add wrapped lucene 84 codec --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 33 ++-- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 163 ++++++++++++++++++ .../services/org.apache.lucene.codecs.Codec | 1 + 3 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 37ca6f39068c0..7eda3732bd0c3 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; import org.apache.lucene.backward_codecs.lucene80.Lucene80Codec; +import org.apache.lucene.backward_codecs.lucene84.Lucene84Codec; import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec; import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.Codec; @@ -29,7 +30,9 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Version; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; @@ -124,23 +127,14 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { } public static SegmentInfo wrap(SegmentInfo segmentInfo) { - // special handling for Lucene87Codec (which is currently bundled with Lucene) - // Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs) - Codec codec = segmentInfo.getCodec(); - if (codec instanceof Lucene80Codec) { - codec = new BWCLucene80Codec(); - } else if (codec instanceof Lucene86Codec) { - codec = new BWCLucene86Codec(); - } else if (codec instanceof Lucene87Codec) { - codec = new BWCLucene87Codec(); - } + Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec()); final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed? - org.apache.lucene.util.Version.LATEST, - org.apache.lucene.util.Version.LATEST, + Version.LATEST, + Version.LATEST, segmentInfo.name, segmentInfo.maxDoc(), segmentInfo.getUseCompoundFile(), @@ -155,6 +149,21 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { return segmentInfo1; } + // special handling for Lucene8xCodecs (which are currently bundled with Lucene) + // Use BWCLucene8xCodec instead as that one extends BWCCodec (similar to all other older codecs) + private static Codec getBackwardCompatibleCodec(Codec codec) { + if (codec instanceof Lucene80Codec) { + codec = new BWCLucene80Codec(); + } else if (codec instanceof Lucene84Codec) { + codec = new BWCLucene84Codec(); + } else if (codec instanceof Lucene86Codec) { + codec = new BWCLucene86Codec(); + } else if (codec instanceof Lucene87Codec) { + codec = new BWCLucene87Codec(); + } + return codec; + } + /** * In-memory postings format that shows no postings available. */ diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java new file mode 100644 index 0000000000000..bae26ad6251a9 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +import java.util.Objects; + + +public class BWCLucene84Codec extends BWCCodec { + + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = + new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene84Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = + new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene84Codec.this.getDocValuesFormatForField(field); + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + /** Instantiates a new codec. */ + public BWCLucene84Codec() { + this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + } + + /** + * Instantiates a new codec, specifying the stored fields compression mode to use. + * + * @param mode stored fields compression mode to use for newly flushed/merged segments. + */ + public BWCLucene84Codec(Lucene50StoredFieldsFormat.Mode mode) { + super("BWCLucene84Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode)); + this.defaultFormat = new Lucene84PostingsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return new Lucene60PointsFormat(); + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return KnnVectorsFormat.EMPTY; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 771fc7a0f19b3..684902c1bf71f 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -7,6 +7,7 @@ org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec From 2c53ff6655d7d8c4ed1d42476e669e9af4ae4269 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 10:40:36 +0200 Subject: [PATCH 04/41] apply spot --- .../bwc/codecs/lucene70/Lucene70Codec.java | 1 - .../bwc/codecs/lucene80/BWCLucene80Codec.java | 31 +++++++++---------- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 29 ++++++++--------- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java index 141fbd3447c85..f9ba02676c2d0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java @@ -21,4 +21,3 @@ public Lucene70Codec() { super("Lucene70"); } } - diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index 5b83b3984cd5a..dea916f97425d 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -30,30 +30,27 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; - public class BWCLucene80Codec extends BWCCodec { private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final PostingsFormat postingsFormat = - new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - throw new UnsupportedOperationException("Old codecs can't be used for writing"); - } - }; - - private final DocValuesFormat docValuesFormat = - new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - }; + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + throw new UnsupportedOperationException("Old codecs can't be used for writing"); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); private final StoredFieldsFormat storedFieldsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index bae26ad6251a9..2638c38286f39 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -33,7 +33,6 @@ import java.util.Objects; - public class BWCLucene84Codec extends BWCCodec { private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); @@ -43,21 +42,19 @@ public class BWCLucene84Codec extends BWCCodec { private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PostingsFormat defaultFormat; - private final PostingsFormat postingsFormat = - new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene84Codec.this.getPostingsFormatForField(field); - } - }; - - private final DocValuesFormat docValuesFormat = - new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene84Codec.this.getDocValuesFormatForField(field); - } - }; + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene84Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene84Codec.this.getDocValuesFormatForField(field); + } + }; private final StoredFieldsFormat storedFieldsFormat; From d3f18a53e0e80e4cb420c58d5dcdf8608ac1bd49 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 12:48:01 +0200 Subject: [PATCH 05/41] Add comment --- .../xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index dea916f97425d..a0585c225c713 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -30,6 +30,10 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +/** + * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0 (Elasticsearch [7.0.0,7.6.0) ) + * mounted as archive indices in Elasticsearch 8.x / 9.x. + */ public class BWCLucene80Codec extends BWCCodec { private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); From b3afec7f9cea0f8c6329a9bdd33f51b4f0cddde5 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 12:49:31 +0200 Subject: [PATCH 06/41] Update doc --- .../xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index a0585c225c713..44b89689854e1 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; /** - * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0 (Elasticsearch [7.0.0,7.6.0) ) + * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0 (Elasticsearch [7.0.0-7.5.2] ) * mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene80Codec extends BWCCodec { From 5fef3ebe5380d81eed73bcad755c8796116393e0 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 13:00:06 +0200 Subject: [PATCH 07/41] Add doc --- .../org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java | 2 +- .../xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java | 2 +- .../xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java | 4 ++++ .../xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java | 4 ++++ .../xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java | 4 ++++ 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 7eda3732bd0c3..66399bbc475f9 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -149,7 +149,7 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { return segmentInfo1; } - // special handling for Lucene8xCodecs (which are currently bundled with Lucene) + // Special handling for Lucene8xCodecs (which are currently bundled with Lucene) // Use BWCLucene8xCodec instead as that one extends BWCCodec (similar to all other older codecs) private static Codec getBackwardCompatibleCodec(Codec codec) { if (codec instanceof Lucene80Codec) { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index 44b89689854e1..d7071b3c9df9b 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; /** - * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0 (Elasticsearch [7.0.0-7.5.2] ) + * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 (Elasticsearch [7.0.0-7.5.2]), * mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene80Codec extends BWCCodec { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 2638c38286f39..feda4644bf41b 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -33,6 +33,10 @@ import java.util.Objects; +/** + * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 (Elasticsearch [7.6.0-7.8.0]), + * mounted as archive indices in Elasticsearch 8.x / 9.x. + */ public class BWCLucene84Codec extends BWCCodec { private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 4372e44f8171c..a787ffda8bc38 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -33,6 +33,10 @@ import java.util.Objects; +/** + * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0 (Elasticsearch [7.9.0]), + * mounted as archive indices in Elasticsearch 8.x / 9.x. + */ public class BWCLucene86Codec extends BWCCodec { private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index 7d36811a30626..f0e15caba8d91 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -32,6 +32,10 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +/** + * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.1 (Elasticsearch [7.10.0-7-17.x]), + * mounted as archive indices in Elasticsearch 8.x / 9.x. + */ public class BWCLucene87Codec extends BWCCodec { private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); From 159649553c577dbbdd2b20fcc334bbe4a83becc5 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 13:08:40 +0200 Subject: [PATCH 08/41] update doc --- .../xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java | 4 ++-- .../xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java | 4 ++-- .../xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java | 4 ++-- .../xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index d7071b3c9df9b..f7f1b015b904c 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -31,8 +31,8 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; /** - * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 (Elasticsearch [7.0.0-7.5.2]), - * mounted as archive indices in Elasticsearch 8.x / 9.x. + * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 + * (Elasticsearch [7.0.0-7.5.2]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene80Codec extends BWCCodec { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index feda4644bf41b..690026a2b2b27 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -34,8 +34,8 @@ import java.util.Objects; /** - * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 (Elasticsearch [7.6.0-7.8.0]), - * mounted as archive indices in Elasticsearch 8.x / 9.x. + * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 + * (Elasticsearch [7.6.0-7.8.0]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene84Codec extends BWCCodec { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index a787ffda8bc38..9b91063809f8f 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -34,8 +34,8 @@ import java.util.Objects; /** - * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0 (Elasticsearch [7.9.0]), - * mounted as archive indices in Elasticsearch 8.x / 9.x. + * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2 + * (Elasticsearch [7.9.0-v7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene86Codec extends BWCCodec { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index f0e15caba8d91..d753f56adffc0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -33,8 +33,8 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; /** - * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.1 (Elasticsearch [7.10.0-7-17.x]), - * mounted as archive indices in Elasticsearch 8.x / 9.x. + * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.3 + * (Elasticsearch [7.10.0-7-17.26]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene87Codec extends BWCCodec { From 1ec05016bf624323ec3cbfea02735f617c0e90f9 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 13:11:13 +0200 Subject: [PATCH 09/41] update doc --- .../xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 690026a2b2b27..c8d05392890cb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -35,7 +35,7 @@ /** * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 - * (Elasticsearch [7.6.0-7.8.0]), mounted as archive indices in Elasticsearch 8.x / 9.x. + * (Elasticsearch [7.6.0-7.8.1]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene84Codec extends BWCCodec { From a279780d55fac8f5ec64c13f63bb02fcb416539d Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 14:21:53 +0200 Subject: [PATCH 10/41] Update docs/changelog/119503.yaml --- docs/changelog/119503.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/119503.yaml diff --git a/docs/changelog/119503.yaml b/docs/changelog/119503.yaml new file mode 100644 index 0000000000000..8f16c103dd9c5 --- /dev/null +++ b/docs/changelog/119503.yaml @@ -0,0 +1,5 @@ +pr: 119503 +summary: Support 7x segments as archive in 8x / 9x +area: Search +type: bug +issues: [] From ecb162d12ef349e9a550950b34747373e1c7c4fe Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 14:47:59 +0200 Subject: [PATCH 11/41] update doc --- .../xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 9b91063809f8f..76a103853ed04 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -35,7 +35,7 @@ /** * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2 - * (Elasticsearch [7.9.0-v7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. + * (Elasticsearch [7.9.0-7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene86Codec extends BWCCodec { From 9d9b6af14d5c3f9ff4cdcc72b3f488e2d2c563c8 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 14:51:08 +0200 Subject: [PATCH 12/41] unmute tests --- muted-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index ec47ec92bbac6..3e104e8a5b0bf 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -163,12 +163,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 -- class: org.elasticsearch.upgrades.QueryBuilderBWCIT - method: testQueryBuilderBWC {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/116990 -- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT - method: testQueryBuilderBWC {p0=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/116989 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 From 02ddf8205360b818ba18beba96dc457a6843e70e Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 14:53:03 +0200 Subject: [PATCH 13/41] revert --- muted-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3e104e8a5b0bf..ec47ec92bbac6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -163,6 +163,12 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 +- class: org.elasticsearch.upgrades.QueryBuilderBWCIT + method: testQueryBuilderBWC {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/116990 +- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT + method: testQueryBuilderBWC {p0=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/116989 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 From 528f2205fa06e687a280d038c74b2507a295e8fe Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Tue, 7 Jan 2025 12:12:11 +0200 Subject: [PATCH 14/41] Update docs/changelog/119503.yaml --- docs/changelog/119503.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/119503.yaml b/docs/changelog/119503.yaml index 8f16c103dd9c5..eb4d641906c5b 100644 --- a/docs/changelog/119503.yaml +++ b/docs/changelog/119503.yaml @@ -2,4 +2,5 @@ pr: 119503 summary: Support 7x segments as archive in 8x / 9x area: Search type: bug -issues: [] +issues: + - 117042 From 87f790f91d6251577b931230f1989726ee3683d9 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 9 Jan 2025 13:42:28 +0200 Subject: [PATCH 15/41] Update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 14 +- .../bwc/codecs/lucene70/BWCLucene70Codec.java | 40 ++--- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 89 ++++------- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 140 ++++++----------- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 133 ++++++---------- .../Lucene86MetadataOnlyPointsFormat.java | 62 ++++++++ .../Lucene86MetadataOnlyPointsReader.java | 145 ++++++++++++++++++ .../bwc/codecs/lucene87/BWCLucene87Codec.java | 145 ++++++++---------- 8 files changed, 420 insertions(+), 348 deletions(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 66399bbc475f9..b5a75f5087446 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -21,6 +21,7 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; @@ -50,18 +51,25 @@ protected BWCCodec(String name) { super(name); } + protected final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + throw new UnsupportedOperationException("Old codecs can't be used for writing"); + } + }; + @Override - public NormsFormat normsFormat() { + public final NormsFormat normsFormat() { throw new UnsupportedOperationException(); } @Override - public TermVectorsFormat termVectorsFormat() { + public final TermVectorsFormat termVectorsFormat() { throw new UnsupportedOperationException(); } @Override - public KnnVectorsFormat knnVectorsFormat() { + public final KnnVectorsFormat knnVectorsFormat() { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index 5a49a7a415b9c..f3d80cbcfe842 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -12,6 +12,7 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; @@ -21,7 +22,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; @@ -33,24 +33,13 @@ */ public class BWCLucene70Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; private final StoredFieldsFormat storedFieldsFormat; - private final DocValuesFormat defaultDVFormat = new Lucene70DocValuesFormat(); - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - }; - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - throw new IllegalStateException("This codec should only be used for reading, not writing"); - } - }; + private final LiveDocsFormat liveDocsFormat; + private final CompoundFormat compoundFormat; + private final DocValuesFormat docValuesFormat; + private final PointsFormat pointsFormat; // Needed for SPI loading @SuppressWarnings("unused") @@ -60,7 +49,18 @@ public BWCLucene70Codec() { protected BWCLucene70Codec(String name) { super(name); - storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + this.segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.liveDocsFormat = new Lucene50LiveDocsFormat(); + this.compoundFormat = new Lucene50CompoundFormat(); + this.docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return new Lucene80DocValuesFormat(); + } + }; + this.pointsFormat = new Lucene60MetadataOnlyPointsFormat(); } @Override @@ -100,6 +100,6 @@ public PostingsFormat postingsFormat() { @Override public PointsFormat pointsFormat() { - return new Lucene60MetadataOnlyPointsFormat(); + return pointsFormat; } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index f7f1b015b904c..04e33cc983ed7 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -10,25 +10,20 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat; import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; -import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; /** * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 @@ -36,48 +31,34 @@ */ public class BWCLucene80Codec extends BWCCodec { - private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - throw new UnsupportedOperationException("Old codecs can't be used for writing"); - } - }; - - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - }; - private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); - + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; private final StoredFieldsFormat storedFieldsFormat; + private final LiveDocsFormat liveDocsFormat; + private final CompoundFormat compoundFormat; + private final DocValuesFormat docValuesFormat; + private final PointsFormat pointsFormat; - /** Instantiates a new codec. */ + // Needed for SPI loading + @SuppressWarnings("unused") public BWCLucene80Codec() { - super("BWCLucene80Codec"); - this.storedFieldsFormat = new Lucene50StoredFieldsFormat(); - } - - @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - - @Override - public TermVectorsFormat termVectorsFormat() { - return vectorsFormat; + this("BWCLucene80Codec"); } - @Override - public final PostingsFormat postingsFormat() { - return postingsFormat; + public BWCLucene80Codec(String name) { + super(name); + this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + this.segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.liveDocsFormat = new Lucene50LiveDocsFormat(); + this.compoundFormat = new Lucene50CompoundFormat(); + this.docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return new Lucene80DocValuesFormat(); + } + }; + this.pointsFormat = new Lucene60MetadataOnlyPointsFormat(); } @Override @@ -90,6 +71,11 @@ public final SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -100,25 +86,18 @@ public final CompoundFormat compoundFormat() { return compoundFormat; } - @Override - public final PointsFormat pointsFormat() { - return new Lucene60PointsFormat(); - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - private final NormsFormat normsFormat = new Lucene80NormsFormat(); - @Override - public final NormsFormat normsFormat() { - return normsFormat; + public final PostingsFormat postingsFormat() { + return postingsFormat; } @Override - public final KnnVectorsFormat knnVectorsFormat() { - return KnnVectorsFormat.EMPTY; + public final PointsFormat pointsFormat() { + return pointsFormat; } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index c8d05392890cb..41f41503a13bb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -10,28 +10,22 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat; import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; -import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; - -import java.util.Objects; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; /** * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 @@ -39,58 +33,41 @@ */ public class BWCLucene84Codec extends BWCCodec { - private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final PostingsFormat defaultFormat; - - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene84Codec.this.getPostingsFormatForField(field); - } - }; - - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene84Codec.this.getDocValuesFormatForField(field); - } - }; - + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; private final StoredFieldsFormat storedFieldsFormat; - - /** Instantiates a new codec. */ + private final LiveDocsFormat liveDocsFormat; + private final CompoundFormat compoundFormat; + private final DocValuesFormat docValuesFormat; + private final PostingsFormat postingsFormat; + private final PointsFormat pointsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") public BWCLucene84Codec() { - this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - } - - /** - * Instantiates a new codec, specifying the stored fields compression mode to use. - * - * @param mode stored fields compression mode to use for newly flushed/merged segments. - */ - public BWCLucene84Codec(Lucene50StoredFieldsFormat.Mode mode) { - super("BWCLucene84Codec"); - this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode)); - this.defaultFormat = new Lucene84PostingsFormat(); - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - - @Override - public TermVectorsFormat termVectorsFormat() { - return vectorsFormat; + this("BWCLucene84Codec"); } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + public BWCLucene84Codec(String name) { + super(name); + this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + this.segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.liveDocsFormat = new Lucene50LiveDocsFormat(); + this.compoundFormat = new Lucene50CompoundFormat(); + this.docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return new Lucene80DocValuesFormat(); + } + }; + this.postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return new Lucene84PostingsFormat(); + } + }; + this.pointsFormat = new Lucene60MetadataOnlyPointsFormat(); } @Override @@ -103,6 +80,11 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -113,52 +95,18 @@ public CompoundFormat compoundFormat() { return compoundFormat; } - @Override - public PointsFormat pointsFormat() { - return new Lucene60PointsFormat(); - } - - @Override - public KnnVectorsFormat knnVectorsFormat() { - return KnnVectorsFormat.EMPTY; - } - - /** - * Returns the postings format that should be used for writing new segments of field. - * - *

The default implementation always returns "Lucene84". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - - /** - * Returns the docvalues format that should be used for writing new segments of field - * . - * - *

The default implementation always returns "Lucene80". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); - - private final NormsFormat normsFormat = new Lucene80NormsFormat(); + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } @Override - public NormsFormat normsFormat() { - return normsFormat; + public PointsFormat pointsFormat() { + return pointsFormat; } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 76a103853ed04..b4b9eb2e84dc0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -10,79 +10,64 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; -import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat; import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; -import java.util.Objects; - /** * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2 * (Elasticsearch [7.9.0-7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene86Codec extends BWCCodec { - private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final PointsFormat pointsFormat = new Lucene86PointsFormat(); - private final PostingsFormat defaultFormat; - - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene86Codec.this.getPostingsFormatForField(field); - } - }; - - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene86Codec.this.getDocValuesFormatForField(field); - } - }; - + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; private final StoredFieldsFormat storedFieldsFormat; - - /** Instantiates a new codec. */ + private final LiveDocsFormat liveDocsFormat; + private final CompoundFormat compoundFormat; + private final DocValuesFormat docValuesFormat; + private final PostingsFormat postingsFormat; + private final PointsFormat pointsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") public BWCLucene86Codec() { - super("BWCLucene86Codec"); - this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(Lucene50StoredFieldsFormat.Mode.BEST_SPEED)); - this.defaultFormat = new Lucene84PostingsFormat(); - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + this("BWCLucene86Codec"); } - @Override - public TermVectorsFormat termVectorsFormat() { - return vectorsFormat; - } - - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + /** Instantiates a new codec. */ + public BWCLucene86Codec(String name) { + super(name); + this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + this.segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.liveDocsFormat = new Lucene50LiveDocsFormat(); + this.compoundFormat = new Lucene50CompoundFormat(); + this.docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return new Lucene80DocValuesFormat(); + } + }; + this.postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return new Lucene84PostingsFormat(); + } + }; + this.pointsFormat = new Lucene86MetadataOnlyPointsFormat(); } @Override @@ -95,6 +80,11 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -105,53 +95,18 @@ public CompoundFormat compoundFormat() { return compoundFormat; } - @Override - public PointsFormat pointsFormat() { - return pointsFormat; - } - - @Override - public final KnnVectorsFormat knnVectorsFormat() { - return KnnVectorsFormat.EMPTY; - } - - /** - * Returns the postings format that should be used for writing new segments of field. - * - *

The default implementation always returns "Lucene84". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - - /** - * Returns the docvalues format that should be used for writing new segments of field - * . - * - *

The default implementation always returns "Lucene80". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); - - private final NormsFormat normsFormat = new Lucene80NormsFormat(); - @Override - public NormsFormat normsFormat() { - return normsFormat; + public PostingsFormat postingsFormat() { + return postingsFormat; } + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java new file mode 100644 index 0000000000000..b59be6e41b5fe --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java @@ -0,0 +1,62 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.codecs.PointsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * Allows reading metadata only from Lucene 8.6 point format + **/ +public class Lucene86MetadataOnlyPointsFormat extends PointsFormat { + + static final String DATA_CODEC_NAME = "Lucene86PointsFormatData"; + static final String META_CODEC_NAME = "Lucene86PointsFormatMeta"; + + /** Filename extension for the leaf blocks */ + public static final String DATA_EXTENSION = "dim"; + + /** Filename extension for the index per field */ + public static final String INDEX_EXTENSION = "dii"; + + static final int DATA_VERSION_START = 0; + static final int DATA_VERSION_CURRENT = DATA_VERSION_START; + + static final int INDEX_VERSION_START = 0; + static final int INDEX_VERSION_CURRENT = INDEX_VERSION_START; + + /** Sole constructor */ + public Lucene86MetadataOnlyPointsFormat() {} + + @Override + public PointsWriter fieldsWriter(SegmentWriteState state) { + throw new UnsupportedOperationException("Old codecs may only be used for reading"); + } + + @Override + public PointsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene86MetadataOnlyPointsReader(state); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java new file mode 100644 index 0000000000000..d8c238aae8d3e --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -0,0 +1,145 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.IndexInput; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** Reads the metadata of point values previously written with Lucene86PointsWriter */ +public final class Lucene86MetadataOnlyPointsReader extends PointsReader { + final IndexInput dataIn; + final SegmentReadState readState; + final Map readers = new HashMap<>(); + + /** Sole constructor */ + public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException { + this.readState = readState; + + String indexFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.INDEX_EXTENSION + ); + + Map fieldToFileOffset = new HashMap<>(); + + // Read index file + try (ChecksumIndexInput indexIn = EndiannessReverserUtil.openChecksumInput(readState.directory, indexFileName, readState.context)) { + Throwable priorE = null; + try { + CodecUtil.checkIndexHeader( + indexIn, + Lucene86MetadataOnlyPointsFormat.META_CODEC_NAME, + Lucene86MetadataOnlyPointsFormat.INDEX_VERSION_START, + Lucene86MetadataOnlyPointsFormat.INDEX_VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + int count = indexIn.readVInt(); + for (int i = 0; i < count; i++) { + int fieldNumber = indexIn.readVInt(); + long fp = indexIn.readVLong(); + fieldToFileOffset.put(fieldNumber, fp); + } + } catch (Throwable t) { + priorE = t; + } finally { + CodecUtil.checkFooter(indexIn, priorE); + } + } + + String dataFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.DATA_EXTENSION + ); + boolean success = false; + dataIn = EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context); + try { + + CodecUtil.checkIndexHeader( + dataIn, + Lucene86MetadataOnlyPointsFormat.DATA_CODEC_NAME, + Lucene86MetadataOnlyPointsFormat.DATA_VERSION_START, + Lucene86MetadataOnlyPointsFormat.DATA_VERSION_START, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + + // NOTE: data file is too costly to verify checksum against all the bytes on open, + // but for now we at least verify proper structure of the checksum footer: which looks + // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption + // such as file truncation. + CodecUtil.retrieveChecksum(dataIn); + + for (Map.Entry ent : fieldToFileOffset.entrySet()) { + int fieldNumber = ent.getKey(); + long fp = ent.getValue(); + dataIn.seek(fp); + PointValues reader = new MetadataOnlyBKDReader(dataIn); + readers.put(fieldNumber, reader); + } + + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public PointValues getValues(String fieldName) { + FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName); + if (fieldInfo == null) { + throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized"); + } + if (fieldInfo.getPointDimensionCount() == 0) { + throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values"); + } + + return readers.get(fieldInfo.number); + } + + @Override + public void checkIntegrity() throws IOException { + CodecUtil.checksumEntireFile(dataIn); + } + + @Override + public void close() throws IOException { + dataIn.close(); + // Free up heap: + readers.clear(); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index d753f56adffc0..627c013d2dceb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -9,28 +9,25 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; -import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat; import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.Lucene86MetadataOnlyPointsFormat; + +import java.util.Objects; /** * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.3 @@ -38,51 +35,59 @@ */ public class BWCLucene87Codec extends BWCCodec { - private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final PointsFormat pointsFormat = new Lucene86PointsFormat(); - private final PostingsFormat defaultFormat; - - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene87Codec.this.getPostingsFormatForField(field); - } - }; + private enum Mode { + /** Trade compression ratio for retrieval speed. */ + BEST_SPEED(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, Lucene80DocValuesFormat.Mode.BEST_SPEED), + /** Trade retrieval speed for compression ratio. */ + BEST_COMPRESSION(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene87Codec.this.getDocValuesFormatForField(field); - } - }; + /** compression mode for stored fields */ + private final Lucene87StoredFieldsFormat.Mode storedMode; - private final StoredFieldsFormat storedFieldsFormat; + /** compression mode for doc value fields */ + private final Lucene80DocValuesFormat.Mode dvMode; - /** Instantiates a new codec. */ - public BWCLucene87Codec() { - super("BWCLucene87Codec"); - this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); - this.defaultFormat = new Lucene84PostingsFormat(); - this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + Mode(Lucene87StoredFieldsFormat.Mode storedMode, Lucene80DocValuesFormat.Mode dvMode) { + this.storedMode = Objects.requireNonNull(storedMode); + this.dvMode = Objects.requireNonNull(dvMode); + } } - @Override - public TermVectorsFormat termVectorsFormat() { - return vectorsFormat; + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; + private final StoredFieldsFormat storedFieldsFormat; + private final LiveDocsFormat liveDocsFormat; + private final CompoundFormat compoundFormat; + private final DocValuesFormat docValuesFormat; + private final PostingsFormat postingsFormat; + private final PointsFormat pointsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene87Codec() { + this("BWCLucene87Codec", Mode.BEST_COMPRESSION); } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + public BWCLucene87Codec(String name, Mode mode) { + super(name); + this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + this.segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(mode.storedMode); + this.liveDocsFormat = new Lucene50LiveDocsFormat(); + this.compoundFormat = new Lucene50CompoundFormat(); + this.docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return new Lucene80DocValuesFormat(mode.dvMode); + } + }; + this.postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return new Lucene84PostingsFormat(); + } + }; + this.pointsFormat = new Lucene86MetadataOnlyPointsFormat(); } @Override @@ -95,6 +100,11 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -105,53 +115,18 @@ public CompoundFormat compoundFormat() { return compoundFormat; } - @Override - public PointsFormat pointsFormat() { - return pointsFormat; - } - - @Override - public final KnnVectorsFormat knnVectorsFormat() { - return KnnVectorsFormat.EMPTY; - } - - /** - * Returns the postings format that should be used for writing new segments of field. - * - *

The default implementation always returns "Lucene84". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - - /** - * Returns the docvalues format that should be used for writing new segments of field - * . - * - *

The default implementation always returns "Lucene80". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - private final DocValuesFormat defaultDVFormat; - - private final NormsFormat normsFormat = new Lucene80NormsFormat(); - @Override - public NormsFormat normsFormat() { - return normsFormat; + public PostingsFormat postingsFormat() { + return postingsFormat; } + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } } From d91844d63addca48f7de13fa45481711e1b64ef6 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 9 Jan 2025 17:16:40 +0200 Subject: [PATCH 16/41] Add ITests for LuceceCodecs8.x --- .../AbstractUpgradeCompatibilityTestCase.java | 2 +- .../archiveindex/RestoreFromLucene80IT.java | 25 ++++++++++++++++++ .../archiveindex/RestoreFromLucene84IT.java | 25 ++++++++++++++++++ .../archiveindex/RestoreFromLucene86IT.java | 25 ++++++++++++++++++ .../archiveindex/RestoreFromLucene87IT.java | 25 ++++++++++++++++++ .../resources/snapshot_vlucene80.zip | Bin 0 -> 13405 bytes .../resources/snapshot_vlucene84.zip | Bin 0 -> 7324 bytes .../resources/snapshot_vlucene86.zip | Bin 0 -> 10503 bytes .../resources/snapshot_vlucene87.zip | Bin 0 -> 6612 bytes 9 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene80IT.java create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene84IT.java create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene86IT.java create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene87IT.java create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene80.zip create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene84.zip create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene86.zip create mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene87.zip diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java index 83473e7b990d4..9b5ea8a0d98af 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java @@ -211,7 +211,7 @@ private static void unzip(Path zipFilePath, Path outputDir) throws IOException { } } - public final void testArchiveIndex() throws Exception { + public final void testIndex() throws Exception { verifyCompatibility(indexCreatedVersion); } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene80IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene80IT.java new file mode 100644 index 0000000000000..fe5c446ab997f --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene80IT.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.test.cluster.util.Version; + +/** + * Test case restoring snapshot created with luceneCoded8.0 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.0.0 -> LuceneVersion 8.0.0 -> LuceneCodec lucene80 + * 3. Added 5 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene80.zip) + * 4. Index Restored to version: Current-1 : 8.x + * 5. Cluster upgraded to version: Current : 9.x + */ +public class RestoreFromLucene80IT extends ArchiveIndexTestCase { + + public RestoreFromLucene80IT(Version version) { + super(version, "lucene80"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene84IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene84IT.java new file mode 100644 index 0000000000000..e95d9f1c96662 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene84IT.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.test.cluster.util.Version; + +/** + * Test case restoring snapshot created with luceneCoded8.4 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.6.0 -> LuceneVersion 8.4.0 -> LuceneCodec lucene84 + * 3. Added 5 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene84.zip) + * 4. Index Restored to version: Current-1 : 8.x + * 5. Cluster upgraded to version: Current : 9.x + */ +public class RestoreFromLucene84IT extends ArchiveIndexTestCase { + + public RestoreFromLucene84IT(Version version) { + super(version, "lucene84"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene86IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene86IT.java new file mode 100644 index 0000000000000..deb541846a6aa --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene86IT.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.test.cluster.util.Version; + +/** + * Test case restoring snapshot created with luceneCoded8.6 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.9.0 -> LuceneVersion 8.6.0 -> LuceneCodec lucene86 + * 3. Added 5 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene86.zip) + * 4. Index Restored to version: Current-1 : 8.x + * 5. Cluster upgraded to version: Current : 9.x + */ +public class RestoreFromLucene86IT extends ArchiveIndexTestCase { + + public RestoreFromLucene86IT(Version version) { + super(version, "lucene86"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene87IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene87IT.java new file mode 100644 index 0000000000000..18059e617f2a5 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromLucene87IT.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.test.cluster.util.Version; + +/** + * Test case restoring snapshot created with luceneCoded8.7 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.10.0 -> LuceneVersion 8.7.0 -> LuceneCodec lucene87 + * 3. Added 5 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene87.zip) + * 4. Index Restored to version: Current-1 : 8.x + * 5. Cluster upgraded to version: Current : 9.x + */ +public class RestoreFromLucene87IT extends ArchiveIndexTestCase { + + public RestoreFromLucene87IT(Version version) { + super(version, "lucene87"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene80.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene80.zip new file mode 100644 index 0000000000000000000000000000000000000000..d5740ebc99b4fd7fefe4259cf9bf1790ac9af5a3 GIT binary patch literal 13405 zcmd6NbyQXB*8ZkDq+7bAJCp`V$&E;NcXxM)bV^AhB_Sz|Y`Qz7K|~~^pZ|K?VvM3jhGX1G0SS)VEWp9yo)K3c&*oKm;(e zHZ*c&WC5xm0H7kKd`xbKlPeMc0{S}y000d>3s9m;KPbaMJJ`car}{3uOtn|0Uk>=r zABldOHX+eCA0SSTavzya6-Fh*7+h1yC!OL(mjWsh ziYpfGA4g*Tn(7!3?Bj%dO9*n-bncv-oP-1@+)3MPiQEVV zoC`d7Apey%lck=ck%Qx}ayQWR1^f{T5qu%wj|%+pU6V6A8CU~1#&04^znn1n1~O?AM6@F`aGnq)0@0jixDa}aE?h#cr$Rb#Ab zGe=@IeRsB5@ycWnTcq>T*%}v@wUifqqf|lEoqmgm*#)BxU#d@(&h56ceT1`5L`Xl9 zq)Ns+AB(<3R}~s$#K#*5@!lIa@MhmTS$r{I1k0`QGGgXU8?L~CEsV&K(Z{c>AH|rE zU65&IjFFb?E0;E$y|)*w#=199k(&6nScPOJh2a!YQ^r2p3@%8{%DXg-FB4r<%#>`$ zGJrbTQf;kOkqZ%h-li^HHmJgB9`Z~^vUZ&FG|_#McJ%}jF?^EU{(AgPg{N>l&}BpA zaP#YAq)=Ya2PeqW)(#?JLe`6={^drWy7Dqj6xlDUvGc7~XcwLfHZklu1EujkrzWR! z38mcL&F|g6?Yb;4ikxEBj-QJ3j}>XAR`dfuq8$%ey?H{ptG7gcw89xKi>p(NF-wMJ z7jh~CT8a-7C!p=ypT)!(QH7#jm|6uv%NFD)mD%voQ|irB|~1tWD3 zn#)%qJUl*vz|hQ1$QvS0Mf3ggmGU?cn9|oh#D*Efrd^>NcUaYMZYKMZz6V0SXJ)bL zi}BeEh}T`QhYmsx0R*w1h5*;5h&KX0@4_UiMc*DP28AM8k&DC8Bk62#+)xdDYfD|_ zFzTU2a9F$mLVgaXaFcyH-$ne&S1C8=?x$`yLCeiq2}Uuj9q7gepzx3}!QP>V0`dBb zGGV4KRVr!>oMZSCdiBXu5%zOufatHonP`&eg>hw@`5)4@MBy4bo~}}9CWV71vi7lgiumZxcC|U<0joMC)#dDt4U!H86uD0OPJ{I z{9gF!n^kYBdYg}EE6mMc&#sa9~_ zJbpxCpr+i*-mAIU{!1aFKS>fJ9V0Hu&^M+dDQUu_la`jVL(|h!-?PFF%l4U1$EJLB z2ecSo@WS!!B2j({n9Zc_tPP9kpL3ErR+Nv7)o27rP(yCOY;XhqXr+zY`imDu5sp*u zl5cilnI$e*4J^Qe@;=<5qr>q}HvJERQq__{!SqErH9a-S+!{SSH6tzc`ASMs;t&A3 zzdf<>!;Qda^de8&1mlnXHKeSlu(Y~j8@{KJnzCnIZv|4^^04P$gc*go8x^_Hor#NF zi-&=Dix3CT#9_|F!>Cfu+#V}z=jfA$fQCaKwhsj%R1y%F5fUD*`)a}9Ru8iev+E8q zPs%Vz59-SG%dxUQfUsboA9w`nmlK~DP)*lSk5BJSO;lD-Z-W_!Nx)_qW9JxV=NK7) z;N{@23dGEPUq{LM-gjzve!9PNdbo3bymQnyg>zA|oF@)NlDZ&Iq)=bNO;RC+06J+6 zH!r`=D1Rdtt8pE7ojP%AeWM`Pp0^%0q!SLVuAza2XgK*25{D0^TxSR3C&lZH-R=I( zqO<=m6`k|{e$j_JdqPtf7ZsR!OYubV-a!_v$o_88X$%4~f1YNz1qI>|oswi?0E}ksHe~kit4B)oeSMfq>l54ZDvad zMMJ7lUlVY~RUr%lIZ%WYRj{*gE@uvol%n!Ss08!L=(rkb31|BZ;ge4>BdBg9YqG)y zjeysev|ceK@90#Ge8>(KLp8#gLU)c>u=?oiPY{IbJR;8@#8nKlG$JFMB<6t7am*xe zRHUg)a1bTmL-JYGlSD@XzY+UeOe94Oqv52EBDex5Eac7xDfb1dlA&n!8$y@gf202( zmoR-C#*Dm!@g3@;!^HQP;-F`0`2>!Q&{(I}jPQUeY4!)slrW5Z5gYS}%{g zv?8Wlt6joPA085M%O<;CbM`+${&<9sADr95H-(jNR=V<_;5jBdjLIkU-VsdE$IKAL z!4ROL*nax%;f|R|djB9n@5U%?u9dLp&VE;-7cWkh18k-W4g2j&=~K>77mLj47N%c$ zQ^YMKdsJB|Uu?W&)ZlMioULv4_&Ar<6V2e^yxm+@*g(YSaomNFIf~HRS8ot>HoV{u%Ekz9ZDFpbtC`lv@c1CAxzizz%)D-iXHz z;>}NxF2c$fW5bt_!thJ=FaJ!B7PyGV&t+Mf;&bW!70J3{cYweSBksFbojOdUO8(B{ z$JXV_<&Q2OTcw^!T3=VBWKR3S4lG`NWi+KPLVUd?bzEo3H-;dqk64mIOdD!de*%B5 zzIG0`RWz%-wY^YPZZh=Y>g*^f-%RA|;9~+^>SU9!G462tLsMG@fub`(*GXqXiE!g0 zQ2B&Fp2W}qDbToHMnW#$CO~r35NAge^j7+^4m){Td$$d=m9z!ZQdSEaQFY1LS|{|~ zsRl}g|1;p`V%2spB$LVEY4uco?diHj!d&mSQ|J{=Sw&UXAkmt%SuAFLg_7-8^znyKe*IG@%|E4QYP`}(O0AEp5ycUxgrD4oHE< zHjMN~bgtUUTW^ET6fwCD|QezSYHgtU9fSF6lM1EkL;NQEM-ILV%)@0Y0fW)6% z=amlf1CvImiHC@eQqnMi1J>}^$ij!T;|j;Bh1su3<*3k6X&$PKxp5?639avmtkaO( z%3H7TtF)_Jw4X{v?bjrc1ZGdU`=ni> z!ovangOvEM7rr`sm=*+CQtx@~4P1OU$-gkD)$0;^MuRgGJ8KaAa)!5WtBVBIQOaqK z#^>6-`J(3>_k>%!;R@iWyhCm6)$06YjBkU)SvH!dx%Sx;-Ab8s4GX&5GSM9hMc#as zt|SII1`Ta3WsUba-bSJZW5pX2j24puODy}?fqiu@0&O`H$A(s3hQnlruXx^Amlt_3 zRPwg7ly8jko}**G>lZa!IH18hS@MnkkU5<%mws_YB*KA=A&O2TiK8sEKs7xQ-Pg%y ztPY)0#%7#J{fs&(A%Izi?DXb2e^XpA!)f$nYi*&4`wI*`oP%#9wT~b|^C{u=>@HVa zUP#0}co7W0!lH|nZH7Wfte8a)`BN6iGdue2U+IZ7$a?`QX1tjEZYL*?@ z-S_zN>sXWx&Xw z68m#71rXz9ya12nfFZ4va%d3ke57pRbbP_pfE5%^|8jit@PJDR6WhhaIx2pm`7!>X zTPQ#TcUg9UL7cbNS{AY8G zhXnrY>b$y490gJot^9jDs{#@L zGZa;U+JSfzS%M)+*xvj2` z_5hW3EiR+ns^DaD-5Mg7J!?>Od7jOg3lVdQAlCWYNWsvNw~w);*ipF7T}j97eR)1$ zX08d>Yp`F7E>jewr=;~6_S+>xvJOzuaP|df(|dj6oZrc$l+Jd^@13=Cg-;0N+DLvg z&HH8~!WfHD8qXP#e{)NWeoIuGUz}*Fy0m9S4CU(_ZF4hjMxt$EUjzin)Yt{Q`nnTEWHS&m2ItjaIg8NU#HyMaJQWKSnr%Tt`B~V^yi$% zvzZEcdK8)XALS@Tjwy~+u@R084#k+06Z^qxG zB^IN&Z3x8LlR%=72(q%rNf%+Vpo+wXJ*AQh=|sWuM4M1Y4fEV$wA0~5M9W5xkwA7* zo$}^jgT%j-EFcKKyoE(dw|HB8`-+LMm{@}gKh{A& zykvw)UGlBycR$jH7~$=6;S_T+&hd%jE{V@oRDpZ5TFQ+k%pS7^4v!F6g|7|&Zby%31C=A~IXKQS=$aDi`ITblY9J9o*^;SoR zy&gK-Xw@-xT?p!xu+@-gYSf2T;X!-hIq=Urfn7UZ&p}Q(k0KDGhDCruJ70F8);vO1 zHza9b9^pdOKaRL;s+yi>^dRb16M5s=fy^JB;kJMJ@Ge7s4%B@5o2&j`_U?b9)iUva ze%0UKK0STA>K9_>WlX@9#5NJ(>54pfmBq#h&jezYmb2@V9gw1P?qlwiV;Jchke6Dw zw?{z@LqoN7a?<0;Vm*Ula6BfokodXP5>jie+5da8#QiIt{Y_bN`csy$|Bxk7wWEVx zrL1&nTC0JFXspxiq~M!01(sBev4*p)H^!^tsHG$l8xzDn` z59>~M2Ku91hYD*lY!2Bg4y0HH@Zq=Gt#S#aqD32_t^EQqa9|A)yV`g}MyALYkxK|h zbl8&Kw^zrnS037o-mfA>Iop3zFO{KnkU4-ax(oVqk$tXYfRM}uAG>XEY?bcJ z9yi#34)>roX}Y8{4mU_sI(wLe5L>BOLCl?2zj;ehgBitjD?5nC##!4lt>5?wrHxC< z>picfJue@hre(pyQaMbR0QI9+&hr^*?~zoUBVD1-WXy@wuTNSoC`^uT4zB!cY_5ER zy>_3KdCgW4rw$}@WIC~TeiwN(Z3tw5l)%#onvj@wN_!PIwOsIA2X!lg``X6hgoy^o zxtpS$ksn>ST|~fqMc+~^_99t)z@kR_woHD1uJp`@ml`kJSrPH{bcKoZ{N@>Vk`rInyksQ$eo>J6=CeSPAWjmdwA{N|rtgDg3*S+q z5_!=8d!V9i6FTO%Kt9O_#a(zg(r+u9>n9grZkkRtU$_}~ebLcUvZ{dp3 zgIq7d0xZB#*0_m~l1jQ*J%{VGBeuXxd$R)6;WYV<{-?q^@Jp^wjy?utN3((_D?D-1 z@ZZ=GbTMTtJ(G-@h~kV|h1#K3@mk_pkEzJ4le>Ye1sGbV0Y(Ltj|-Y`i>ZLU=@|zT zhOcr>lFu}lA{n2;uP6E0>iP!?q~j8s3!H;U@^yyH)|JyH>M*Xt^F@7_8Cb_QTY=(Ph@RgZqRKDh|;qbBZNqyhK zu>4wR5#d{In+=+_i0L;g#>VO00@$Q$y@kc@Mz2vWMrIEELtI`DykJc5^6VncLPsUA zNRrCi;8B;Y$`oWT!4qB_1sp7Fm{S~|^Yr=WY;+3*>HM!b#W&rvC^TK5OiXV1JI(EP->=k5Bg0C4LK zyq9v{?mf@n6@5_g&;2Q-{?^u35QAm?&c)sv)R|ZLO_?0EkNlA%7U! zRxBIed|S>Cc{6)+?c$&|A6kIfX*4XJt}opp`hYKIE`e^KajUu#3X250F0mPG1;W}I z4b4U;jY%%G3GSN7MgmJ}&lKRoC?}4Eq4tVzoC^DWb;ojKLkZb}Y z4y!@!-neFX`V`tdAE^UH8m^&^wvv)vFnS~q$kMFYW3kqe6F0O}k}z{rX5r;#1m zm$Lvo-}r~IV91709II@%EUu2OhyWTlmwZ`Ttgz(D$YUTVzvgva7_8+C53)2CkC+gOWYwlw++#QeIm2~(QC5Na;czp<7$6~2Gz^NRkmr z;ta7v6{e`7>MyX%lPONqLH4*XT-2M-;j14X9ROwdl7+fwQl5Du6^|BkVGenbkjcgt zH|OsRx#>TLlc+pFkTsHIcO{D$~-9uFl>th%=3YociBiXm(xU<`0I z7T3zL!$?_?;d$R)I#9-aWEl9y^m9WZ$iB7|Cl-2pKQN!EQw2L7wgY;G6W5KaRBuI; z`eIt5{?SA?f7ZTgjXLl5j*`r4F2fxUudmHs*+mTs5PVNH*B~cLvY#PBUCffzC+f!} zMi&fR-HcOT-HddPBZ@OpARtygyBsS**x+?R^Y6EPa`psi#f{6yE}#Mz5)*K{B)3r88|1E=`u;jKlIge%Bv9 zukn`1W7SwbU`=ul;{@lUAMJAL>}-YmF#qU+X5m=fwqh3ZhKaMA%HRj!{&V6k5cSYoP~WxWJ^A;uLV zgs5}T1b9Jzy50x8+k8xS$pJ+>){CruxsqEcSZD$HeR^nA`DicPcFb+ET_ zYG-!_DvOKe$%l~KNX_cU;%TI%XL$X8te=O+9+#6wx7sV_SY6pcyV?8`?rB$cn<`(dsXzlK@6PduRE$rN@mIMHH?IZ3ukz1!gT9GGk6Z2h z7tx4!){TXMDg1T_{5{cF89D0x4`>GUPovXq6fi02{Y5sK3Ucz0tmseN_A^~uePwVa zF=08WvslG2R`ZJHAk3ZT@?Wp78bkWMLXVPIy&=oE6d!I@bH+s*v|4O(vsp|hAtsr| z$3^aT7J#;@!xCg>1%^im(D{BZfe?dB7F?6-GYmnG>Fh)QR74Kms1Lp<4iynOO+DkUwB=36f6&PW?`b_HL@*;=2 zw+UJ>!VR?E23*e4$;i8-#Mwd=a0Z*%Sn!a$STu;K@CLF;K@i!d`~zB-j=@9}wptRl zHLu4)l~=Jw6Gk8ByNjSZT5>IE;3g!8!6QVG;~UF!9M?B(Yo9(A+K6k!skWH)sA~ul z<(vYiaA0B`_v$d~x9{@6gi?6ldGjGSOzwUo{hv9#|x3ZD^sO>E1 zVvh)33O{XwG35=dd6#5L+1I}T=1W1SV3$qr^2`bL=KP5Ni>Ikve3m}6ny74U2;f}(|xvb^fk4bgI*Cwj& zuBHEQy68q`y{NUkT~t=f!<>ifSPIjYEJKI(AId?Iz$tLI9LY>sOpD+%Z^vyp{lA1z#jLty`FEcCm1aY?&6XSp^LMca2#EFv}2*JLO0fDQ6qv(hY;St>) zgA616VE<6WkgVeJgps91U%w1U_8H^#GNfUy>ZK(elN5uL6B4SEvoi{6MtVw8x^ikP zL1so=W_&UxTB6ElXeDu~eX5lDqT)zx5#ECjn6t`f%3)A{<7>-FbgcO9sI2D=TD>-YYzVE^=^{wm_Xc<%Pe?>$G>DkXyUz{b25zRsX8YZLLr51M{1A_O89{M>G!D zf8B@wQaKjpKg@2w`d$BeAn$s*encvQyB8AJ;dMWxyGEuTkw_5kg#@-W-4E%m=ix^r zEtGp9fgKR{L%M4j_z@`-^ z`yt)kS@;nt6Zc+7;C+VsA>D1K|A>Tze=j6(bNzluciYiFA_)-O3klquz8})vX6}zj z=7je`0=IVWhjh1d^dpkV!+Rlt`$+dgy4x=K5ow3O9 zZznwW%i`|3;>RrBQ2%Wfe>(@gUk-Ptwm;^OOAGk>iSGSC?oJqg1gW9}{QdOtzk$Gj T)er!{1^;V-HRLn>?a}`O-(?K@ literal 0 HcmV?d00001 diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene84.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene84.zip new file mode 100644 index 0000000000000000000000000000000000000000..afa4c9bf8189d911cab8298604b84e447d91022a GIT binary patch literal 7324 zcmb_g2Q-}B)*eCB5sVT%7mi&pzkuz0aemhKWT6003|SQlT8ib>$>&_tF0j z(S`#!2e5aBTl@0x>*?YFFjGf@ZI8^$mjHl)^$`OAz(S7#c*S2L2!)WfKkk%3>MBOm zBny7jthccYcdVPGGjSRUW)AwSe}+V%#&V+ZEfgnoN-iJ40eS4%?5YSkOh-ZVdKJW1 z`eUZmEq59~HQMlSCLf#ZMkHz4hII)e9L1gM!}MmHK}8}a#I{^> zf23Z^b?p4R%=q%|ZOhrfmF*$U7N#eX(e&XY@oI0xp9c2ZEwlC|H0#Zkw@upQ&`_=U zV++2R&WUi*JT_jqux?c^xf1tvx({H*qVAs{p_e6h8Vzz+d6}>)~HN zxL={{t*kxx{)Vae_xSx9f&K>sB~cx3KSecTUA;>>_GwEhm&KWsIbAxBC)4`)kP z4?7o64|JR{6cnKVPaXA3Alb9dA*m0^SXw48PRj~Wx7pN~rVh5-$>eGh+hGs1!RsYf zh{ne`B<}Bjn3e^mpM_0FB$>rg9)_%=F7)hxw%8yvPsW|qa9f!h5DRZU2hNW#AQoPh zCsgd`#R@B&8R-EQ-v$DVxwa!BaBm>J9%Q`bRdG+{dqPuj#Q_PUa%fiR1Y{?5E*3<_ z;_DQ(()+;4JM<;hGf9SN9WsR^N~pVJ!lA%<@`}ZKc-&#Ff~Q`=uq?3o%g(<*nbD+EC}!XR zOK#xb<1T|bCKIS>YQ4jD(m?iHD%}&$FC)RGlf+M(|T z+eY_1@`1Is0W61*q+YEY5+1?Hyq zcX9d4{OambZ|+CfDn}bug2R2p!eL(VA{(7oXk2xSM~z=ftBO2$4Ii0Di3$sEyevei zMqjtH)$H0|4|M0I@p+OxM{svV>^3yZ`_j#63yfVV_fyXrXX0pg8*?$_0uhde-8Xm) zubI$N;S03kMGOpwQ43P^A*UvAl17-eBiNsL+(7acj&7TZPPW!qQ9f|K95^;~tHx4J z%R3C=|gfzJCkozlo{y)-$XQlo-%7pipgdTX4#acV_(Oy(oK(3 zg$a?~m`-E|?u?JlFTYrFF8*A?HAHBBPYIL5C{`%KCi;&2(?Ft0++0Cf4`yubDxix%2+8>cNDAj)g3_w z#6kCiy~k=Pn2GIfdJ2%1DCAz5f#kME_c61_$m)Y8AqB4;_M!U24K_Zsp`5xy4-A|1 zR15Ua?cDy<2f`t0+JaoXT|!!;x&pk@bTvGjQ_sQw_S(uSxTliVgTV?hwCO#6^Ud~r z_vXM_N@cIyc)LjUz=U^+aFKJG4e%{QLF9!D&-m!!=xX)Mg7o1A-0_?`$#PTf$D4I| zGux?wy%KB2(%#X7oh&V7Ug{`0rvxZ=ag;7IX9%@q)8@#IJXwd#(1tG(bw6H=!HyBP z@?*pq!z{A^1+>pux%X1UE;BwC z^zU?YPO-RXKoKZo)#0abcSzmMY-$zEUOQ#jbRD&njSX_HDno#r)D@X_fa#M$!)jb*r>z7uaGI zZC;4o)_yi&|DL8mny0-U(l2Z*vc*gqxwBOSO=B70ARfQ2krsAHC}%+Drd6ZrnOO;G^(AQt0ZACP?a=e$)QH~f z*zaoet8*+5iG}5Q?alC}67}9ExuNwI=#?SC>Tk>#;2a=?$myZ}DW+ zUxAUV?}G1Ez=rOhyF8LzJhjf1RJXZjAFV-PEzt~Ol`z;lJ2cso;{9mUC&=gB?fBKl z`!fcIn8)+bIaE0|6*^_Bl7|meL)*a8P(<57L(xdi$MLrmvOYH$*DfP!=>AT0#2k*`TZHPK9+3F%m)uiW_I$i5T|AS*LKj590wRKf_l3X$RyN2B zuhcZutg*G9WDsiz+?Hgng#ts>HjvsnV}d#!+A-rBNOzFhmiwlTc3qZ9nEZ4YrirQP zSt2`V5C)JTa^Wx?j&Xb~I_NHR^q@;d%E{LPp$S!06?C!E^|m&Iy1_)O6!~FdzpV*> zt;OoswQ!is%DJ3io|!hCX`r2)Vchi(>w(D^EU8czq0$&)CM#`7PP{0)q)oP+yazA- z0%JFgVlq$6ARw_J3+Wp*+}e1d&2X+zM^k;!E)$=jwW~~XJDS_io`YPo#a^?8IKcu> z_XGKWuYIk`X@-Rq*>c;6%=Y1qytKp&ys3L1Wz{zz{a9Xfqr%0FGRmLD<8I-)d$*673(3%3MjTDdv%_-l_fHp{Grl@C${w_^)CAbu+3yns+}d`( zlbml{9ep>|;7UnqlEc!VP1}3^G6ZALfjvWXX2|O0sDU~v`k(^{_{PIl6A8APA0L?b z4AKNKO7AZ|YB2S$?sey1h`MUbCc(&=_W%S>F6BZDa`y1>ww1n{F0UJ_hISf1w{2o3 zq@0$&jugvS66UZ^`IOKdPPQyEO`M*PA;ZMS8&nlf%iBQBWM@)>s*l6vBMtlH`)SZu zXWRANOs}_}{ccA@2uPaZqef0J$tUp%DW-Nm4rH=`+o*hv1t>;c!SfyArFRcn_%0B0 zSldQ6(1%WZ?&Q!SO~R4u`ZR&#Diq$M{=O!(%6nZ|a}Qw6-76_@i0!Y{=27)FM_Gyu?1Oo zVE)#E=J4D|!hHPomTPp^jSc!{Iu1W12X9Gj&Rt)P^c5yLDA=}VBwtykT0Ce$@zqaa zG(mR!Me{5TSd-07H?6$a{41}P9-=Ppzs5~P$qGz`x{pOVO_H7lVxc-Yn*Hvjh444w zA;KjR_>qGl%dq{$Dy@u&`iU|_1ed9DdLJpgQP>eS-_eKncAzABo26;Y#PC*TbtTIb zp?P!CwL{|9O)<33o9=*~!h-xF^)Yfos|;#ZZkb<}cZS@qk~=E>O#3!bv2rp`XIEo@JN&lggpG0#)*FHvu# zlCjK_FU%G_OG~vWjEX42-eoP!PsU!TT`=@pv^g8jLXnod35Qdp%~I;g4KPP6o(?bg zl8j9uzB_L-e-g2cmwGc9l5(u(c>Stmc>4g%`}bE zf?<-OAyjX}`k)`y5OM|ZMm2oP^*oGT{2hag{&X#1aBC^)5;dcwdeo{G`O*{R^A~Z$ zv83OKr`My!dEAK}#fuFfVfL9IbDP zDoI_Sy0wAU%~$Lak7jf^-{c#nnq}IOM!$lt5R;M#zOp%oAK%<7Iio3emA~217>hWj zkg9Ci_B{M-Sd$aLUw;PnnDwU06Vj4MELj%6?dmGMXu6GrSE3YCHqj%JVx9VN0~+Iu zZ0beVTZ|U!$p$MLM>ubCSgKsU%0EH!bmU0=`}vRo5$I0;nApW=wY&NdJ@Uwoq^kdN zrEz~=V`!`dQ7b>O0vC0>tWpcMomfg)yGKa~N>f{6ZW> z-rhsu^&6Q8DzkGBo<8J!^ko9U=saOMwRGq1Ls7m0O3!lamUK0aGyf2d^C`m8$HOsn z-+z7{J@Uwo!tuYZ|5bgnu>IF&S!srvSvSl#O2zmDF1)6F>cq>_4aF;G_q-nC!{bV{ zfy?LKgX@jY_42bI{(EjU9?q9tJ;)xY)>Ji-{guuxwGOBT&%x9wc)tFgBL}Yvw~7}5 zk(akOkSIGVBP~lOGm$zcJ1HkQ9dssD*P%z9M+J%=qx&8UpDg9C{8J+6hd1}M(8>N% zx0)=;B5wyDhJCj(w2I4+Do*1|5AGYb2R#6XdyqK{ZjS)&q)}Mk&OJACcn?k^YaR^JG38ugE|8z>SX6|A^#xo%@kQf*kPYs`smz UV4prpu0-fBA^ORW;^^Q10sOP<{r~^~ literal 0 HcmV?d00001 diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene86.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene86.zip new file mode 100644 index 0000000000000000000000000000000000000000..819c538e00f40f7caf5748ef27f52cf02e28d3eb GIT binary patch literal 10503 zcmb_?1yo#FvvuPhf&}+KfZ*=dxVyW%y9f8+?!g*&2~Myf5L|->Nw7e0hX*rnz6tZb zWHRgjVI5Yp*1dbz-M6~xRGpHOgn&c`006K6#Q;hT_VOZJ7%%{!>VClh@BrpEM#gS5 z^vX(b0Eo;5U(<)>?1lgUgFLyv0`mSoK#<(Btv~(KTircu#7rza+cykDQU<-LL%lK3 z;mo)+dF707cP&97Wb+0|o^Kbt&u*G9NXZka1LIaY>N~~7yY2}5p)_iTq^5g%=guYj z3EH^XLL|Ljt8@+Q7LdHpAVRQtB@=4v@g9bmb0;Rbfku0J%pxnlFwl;RsrxB8oU{GX{2hw}k*#d3pMcka^ z>1mDhf2s=Kj1}gSKV0zvdqKd2M5F*(F&|{W8PJ1NZM@Jh)1C-o^*w_N)SHA< z(rPl_9)c=mvYsFe%!Gp!1k&R~?F(vM2#wCEr3j$4>}<_A?!E1Yx$<-Ysv`<=;lEN! zsNENq2+S#qM$o@_!*59r(lr)9l|a9StBgxW@=q17Zhp?FG8i^4B$$byH^K#T)qunm zPWtrgJGPlXDnkitbbNGhGel{b_7tikWagB1C^L4hI#UpBTs zt#PJH?un@a6sOmw=kD%BzPp0}bW+4`{^^CLC7{?Jdl&IPI|2Pa`qy94Xxm$QZw3MY zc#Qx6aQq6iuCBDZhpD}pn35v1yP%1a6i|gjMOW1HKi`%A62IzIbj{F3v9fJXM>2~< zK`MD>?;XAhc8Q!;FifF4_GXZq*S$!Y=2dGLT*^@QXiVSRQ|}e4xLk>ceR(YZG&i~c zsQDBsSGcMUISNW(Q;@Q^6+MXexZK6Wl{~d=SNm|(t7>TX!`aPg`uB>^=ZFEN~N2M1#i!VxI@!9Ebp3C3zExG)3o!bhr z@yN7EKwuGZPh^xfy&(8Chk>}leWHL+(*ifIc;3`H{-)E3GglPCWQf57_G5nyZa(yA z^8F4UYlf`6i!)JjtZ%Q~2nFLS2xo^W?*dI>dh%?kbBCkWn;LTs^IwrdpbC%^?t3Xi zN~4=D6vr|ZPf;cc$IxN;^7Zjw8pgCKr!!mlU@g^l9uJJc^s{rqP#;~w_hE|m%}j2RVFX6IV-&f1mB7V-cK!)A`D8lH_R*TY0I7tz+esOZj% z#XhNV*qtFP;017p95z;oKdjA26EN8_H_;+vR3pYNy5u=tyaQT@Pb9?Ncwijkw$)9 z(F<3|FAixcu%)V4WAGEWqg)!*-}+dYE#J!UXXtTVa`}oGiPU<3m}IB>sIo4q_0=8M zdk=+)h@~4(cqBJJIT$*kplj8R*i~nuj+A^^s`xJ4f3g^oj~MQ4SVa5A9w)2PW|mp5KH71-WiQS)^zB znUInSxd0^2&8JUF5{jYeI+gM7-nbO96+V5Vipt!IgbIEgc|zuu@BL6g65ar)s(K*Lwc+$-Bv1c_RkRp! zM?V0^b^B9dYYm4nS$73-t5?E7C+WdEqKZI}kmvTNn2lCf$0N;?W^ZVx1O~6e;!jOa zvtyoKKO?BmKU8vRD1l6nAqeb~$flNF}TU~Oo^CY<^j^MTjA^ZLn! zMZP_h_zu6MS0!i%EUWe$3Ec?N@2;{g!s2eN5+NY017UfO^$y~3kV4}Kq}90Jt~@OM zUx_4+Hu`on()!HuMojL`ifqQNQf?ZqiY_t|K+|8zR_UT}=U2^A2WM2?` z1C@>y_GH@XP)?K)&yV=H$|i3gnI44)Jk+MkvU)>xG;>GyD^x^Lbd0UF0P-xnoDOv2x)XJ0h$oSSUJE$jc;2;Rr z=alDJaIq3~yV3#q$n%-imiU8pz4-|^KUk};lM<=GU=Da! z^jzIyM5NB;k_vkXTSXm582om17{o=$1`Ikrb2c?rDr-u&3&@Y}5h8D&0!)iDFy*AT zSjxPp+6-J;9LD6*S>XLordJF4zj6{p3GHTqRoe=XfJ^s$+z+!|x*2P|wCGPJ@-OfQ zC?k-Ji-hwC_rJU9p!PaDnnG@OYfaxgYVq{5PV$NCB6V>d{ zB#yJTt)QNNqV7nfQBgi5-3|*URihO=P(>%o8ga|ZJR^x#qI8zXdNXm0S!8-o9!>s zQpLy?s*5h}zpZ0uRhDoj2lTP`pS(|K+-0yuQc+M3BfiUO3g{rCz11z9Lyf1YIuJ@3 zAoW&QdNb2&dAcK;MI*%GX=qy3tYN9W;!*;?rNo-dDxjE&x6mYf1#F?k$RHEtZ84#Q z%##?m82oUA?>7&UBrDQzYyyxVIxC#^y&ClR0$h=(@aW!rPQZ3y9B{Ug8+0Xb1$$>= zDc?F@k&L)oGIT-SkL9v;k;v-K(H4l#&dVs&8>nP_31)ld-2T;vp|hLM>9&_1&g)|r!C}_ix%jRY&PDpk2*I|1 z@xG;Yw$qcvSXbIC-Q*=NXKyqqw6ET|r?FlG2VR3Zhf8)ege_40q!?9;Li!(@S&>n; zz32$puc$&lnj0(TC`O8i#FO>ccvo&Vv2^b@QwS9Xg_#5r^Fs<_MUtfhvf56+s;=|r zpE+r~mCVm2tpF3pxt5~0n|tBCp>bbxsjU`PV6~zf#7f%0M?c4 zZu-T(6z^M@@DdY$B9p?&Ap`g!(Ayy=;5BbGslsb z(j7w>b|XgC8)op<%7)%_y(>Ea=p}5e22?!ppPmL2!zF}0>n-JTkq&8lLX@v!cBO$zGRkUMd~=oVIQ2!ma@P=Ey6freJDiA&=Eb-!M!WD(#GI!RDqWr z=5|Fzu6kE^+Fg#FUQn$=iCdSo-MBp?iN1v*GM$W{cG-D+;*wZY1=IAQ-1{zkUP zY@Ra`+6)R|x(mN@wo2kaNU%(Di|%YJc3%8+Fz6)Pe}`V{eLe=H>2L+4c^y+5S*%Us z08PsW4gIpf#m>-;Pcb=biTtGM^v%WYQ4+lxK{#TY1U|Bj`0wbNklM4=ty{P-w0BDS zSHwB!K{MK=CQbW0>uDgk7DlASpb0zaNI*7a+@<-hN0MPg3sWh&76oe9Q8%)r6Wu@y zSVhV%^bs0f7oo+hd&4r$&3S#wDxE}O8zWr32Y34_ERlyXz+j1#j1tu*#v`wdM66Lc zDkK0;OG_>PA_cvcjY8g7y(us__$l*x6$SfvNf50(jX1iEL`|d~?Rxi^3fWd+l_NrC zIcRnlAcaP&$wKH1RLmjz%-!1sXYEB&}1_YWi*(#a~ihBr2S~1!? zv)Kv!EMC=iWId(M?NUO;c4!NYxkuK-N?*8=tjf%Br=4vRFu} z+M@(s3DZ?l2Z4bf=hU(Ia`zx21h_%8p*MNGNH{eb8bv+*BiAc}V&wi4k5>9IxV zXb?!*y$Ce%fMHvLH1iUtog__i0dcy=9LTk(B@0M92#+PiWg{l~A|;*)vM{4rP^Ob9 z`S6UAQPOu|%5`l~q`@CX5Wm|!k-LzKDTD{DJOkxG{N^qf&n}eXPd-{k_v-z!-+Rwo zC`u*1BAm&QOrmrNBD{(dIq5IRW>Uc%iNv?ML%e@$H+5w6fp-3#wDhKihB9LIEb>x{ z%*w)s>~?}ecFaF5l>eo)@dB^-2;MBbBVFRBBjZbbP}qqvLdQh{K7k2EqUep;%YmVw zaNmG&S%EZ0VynJ+IRRCSwmNi>#0x0UiW7SLy-vA@^ZKlQ%!&N7LyU4=%XZSxL~O|d zNX953A7|le2wwiy3@F)x{4F0IHk}6I<>sr;vEZOWnE!bp2jjA260z#J8g-Mt|$THPc;# zRPGb)!{Ygs;^B{0m6!^Pp^1r^f)dbLP)X3tOn^pR#_^}l`_Sit`?GRz{Ht^rv|SUz zlffaQnQJJ6G@w(`3Wo%S?e@4QAoSRet2RXKShH#R1*>~rhb|LvNjq-kw$wd~{=IN8 zr(3wM8w`Vb2)jC}J|c*^D3E*MK;$q@wAZHE4u|wk?=5T!hlsl2=(wvS~?$h zk`l2=dp%b3j(3Q9cH8_TFlx9tjf$dq$LG796JCVsE4Rd|ACiwIi^IHNl|rX5Ysm?( zZ5GmS1ISUscE|gDX>Shu%VmAsoD%lFi7t~xIHvYcZ^2Cf!;L5nQotq?{rVltUtL3& zB;DZ@*u87=nM-PAJkp2cf!L%vm930&;{;M`32?Wtu9~74cyyiR1R-gH3FL^|PSR@& zFP1OjlhM0UyUIA@;+$IV(`UIfk^tsDcKU15bAF(u%1x^- zFp{2y!Ftf6T@QR1Y9 zX0rKD3aWE(){g6gfr;xIBC|3krdAm?<|?BdV?jJA`^1tpZ5L@qT(=DYWk@!TK9isH zKQu5h#}^$QmsWouU3-3{W#v;@VfwM20EVU-Pi53yZ1q*n41#SL+A*J@WQGR#fbL#6l+cVTfdbL zG#Z2?QuxV5<`)Jd!m(M81o|x1ktG=8zQZS~!$qRCPi-vUS7rG0l@@^CwsYp_tg7}b z@^izgJWJMmadnhi{pF$PvV(N<Hn-7FAA$mEW)L=?j4jnfOL%cv=f!D`Oi8Ql~@d}Au2Pd$<;-4uz5 z;ME$6AZ~TM(_>vv^Q{}_SK%O?!N&5p{M~f>6dC9>%a*6PFrrm2kTQ|;v%DBS0ci~j+ma&hMZf9p3 zAaJVHdvkMRtA-HrrsLp<_ZxKuC$AX6N%NRb?0={-bx_K@>{mp7zLny|UDyH0_VrWV z6o0jF!ApYt>Ska+I0MRY=;GpH`95I$y=qV%Hc@??g{jnW(YO9xs?axf1_E6yb>-Cs zWtmMBRE%WpEu?;0HT=s|k@}E@xyX_qa>Oy*ty?HI#)|`J52%Y?xdU4{PV{_Aw09=Q z!SKyBn+07`1{r{78Ltf#Bzzc!QpE9J?uU@*#=%XED8VP{j=V`$O#;_vD&>bNA7g&T zzp<8ks6&*j#1Ewkgu>_dCzI<+mg_pQi$~-irV7v)se#yzl3{iSNzUTsUh>Fm$05Y; z`)?=1K%pze;D3(8ipkDlwXXs~wfGGW2EYLYqqi2pq6X-4;ITA{_B_D;mWB!5h)<9G zyMDErrG=9lji9}vsG^mKtf>RNfV_~xPm@Hw%AzH@DtgD4;o;q8XSk?7td9x$mgS*@ zYfU0-WQ)^fept8)i1i6c_9<hM5aFXCys&=lq5iWo?Xwuv z3YS_Ex&Fw!eV(R!(t9>0uHN)8!CbC;937tRpEDHQXR#K2N;aNG<1rN86cjEWS}1eX zpcEX)X1!j%@jc#+=F^hr^SXPl3Ie{>m1&B`KJuwTKQ(G={Bp?`I&s5xdLiW`6sqF= z1)^n)1c=w%d*naCXp|FsQSA<07f&cleO`eX#H57AwOtHPlwe5?s7kY1xvcD15GZAI zagQ!{p}?L$BM95opwpAsen}1%-|9bWaA^kNw(6*!$QH7nkovMa>?K-Xq$EzgdgWdS zqC)@yN3Vk=ECed%rvoU|9w34b*4rhHy*CgiZUgpcl5Ml^RND4*z#)vwW3=kH2~&0; z&nd84__ug=I(f=et4@K zwEz-O%C`cl;VnmmVw$IeWx0YIh~}Hj$*V|w!^Cmnm=Ceh)()$^aqD)arG^fqM!o$~ zot>v@PN5DR?;&gz#9<1aT&RFJR#_u=z)Au8?KuGJ{q1zx6@gDSccpic&vC!f*)2tj zcq&Jm7)otU=kp{`TClC5G(=T>%&ec|tr|HdPl49UJ}T@;upv^y_nA_u&sufhWDZY~ zcC$Ll$Y)Z9)HFV%pIEFbSXeVs;qzv-=noKSFTIJySt(Uy{8+!ZoI;6G+APCNWboc0 z*))+D&>9KPRxORgIyE5fCEn^-A-Yht7YBCP{iWTA| z#WrIabj(zF2~3C4_G<^8*FAICeeG-NNgDI&YKN`COpK6v1yQUq>=_v<9)wGYQxel@~Rf>YXUx| z;@Q!amnWRlywF9SAWwgha%@^%fuItYrohG1H4|;j<#Tzp)rjb(X&N4WSD^cP+z{K` z)^R7>A!RxQ)0}mat+UFo_$z7ME&Sn^`qWp7>Z_QC=k8tf+xg5@RwndnsDWPqh96CJ zQi>N9$8L(#pHn}r47WdVT8GI`B0Wbe$M*s(uoKqLze4HL?OwCSmAFwmcGsDSu)m>@ z+3xM7*~DL@-kbMJYaQ4Vl7%N<|CryBYqRr)aqBsSRLPWIEWd zLA2g$%;pMLl>!*#_f&_seZGcoO&@ov@e~_^yYA!j!+JQZ{r%AN|Dh56_XAU@*E~<& z(IbkGW4!L_>x5F9YZ5XdD-1^`S6)@w6$ANK6IDzY)(`kcoF5tSU3gwL>}-&@1|s4@ zAYLpONU@%OD$$O|%Co?+ba@}pe1}#55o;_`yUnK+@-*tCGj4=@VBF3z=KPE*op076 z;}R35yc(kylka2awCcX*Cwr9$nkdCBtYVkW6on#}IpO2W3_yqiF9+t{c zAk0sGbA5w*FVEk%0ce{N2Xc~N-~;6dne&zc;eEQGe2h<;fAAcMA z_&eC+d51s19$@nQ#O^VF2m8B1CoCjj`%eT9wh(shxpNBGTqm%_pjk+7CfBH|BU7@ z*Li#{|NVo${uP=ZpVI$~{x8>fe3QYrjVSxa0IaedB#F|3^>L)%{Wb>1XtRxxnL=%=h%qp8W#- zAJOec4Vs^s@Ynl)Tu6Oy!YcMJn4qisqq6GP=pWY_-_!HnXMkUL0Y9oden$V-3wXRu t`JUdN5b)2NmR}=zobkUW$shv!bN>Gk2^920jC^|k3kw4PY!N^F_dgV1z?%R7 literal 0 HcmV?d00001 diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene87.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene87.zip new file mode 100644 index 0000000000000000000000000000000000000000..933a6b16120dbed1f72b321f94b22861e8db9db4 GIT binary patch literal 6612 zcmb_gbzD?yw;p<6XpoeU1_1|IqELdw`%=L)F`0s-WYS-rW>6F?_34 zkcl=--4Hv0T!iHqO*SJBjP$532=tIIVp(}}XDjt>#T9u;JkRb#%-u>HNq+3rU1I!v zE|u+8{8xB_=~2-{#qO^4;!)JEIR1483=+m#xHH{|| zzB3wj%u650A00n$t2l52>b;wHTjn+Ym__XTys60ZfSF`LUMl>2T~1Hl3Lm~|B4l|Rd)rP7Y&u(A3mVEAfgox`M1r^R!XXa-}V;8~E`DJ~JyOlDYL zm<0gvzj4WHW9(w)?D8u@O>f);08XIrPU!uzEMIQ^x&XgIAxzDjFZ>N7{O@)7IRe8E z2x^X|s(cWz2-H(s%GA^4y0w;rypZI70gR206O@9=g%cC|J+Q5ri!qP0hma;zP+iKy z-Q3IFOjkrj3ns@4H~u9n;v+VcgkRI~1OxT7F^Uj0894~RAf{UdosT?7+s+5y-3}Q^ ze8V7iotf_lGbX<<7jj_~JRoEn0)7mU0pd1WUVKL6xJV{xVQP%6yt119c{|;EwR83; zof;~41I{KL6YjLr04ya=*j8ubhLKy^Vr>_4J4loUh|g+$w7kto5hOopsxatOba7a1 z7Jbx3xVhf8JK6_VM_)jO+i8`;Imm|>=HP_Y^BICGnhzq~ppRoje`l)YCVo_{kymzK z`Ai%UCt3g%VODLB#c~qIgKTMR?IGg^H}~Rkf-&o$w0_oF+h=()6hf2u@(nWy(az~f zx~sLZOG-9$?_@_D&-N(gj+G8}iJfcl6)0B%#-s%BlVu1HK<=~Cf$}Y>mf?4)8H<$b z%EB@t1&7c^tw5vI>F`~_P6K%w6KZAxe$B#?%BXNnq)*`Oo1b4Zb8RjaUEV;HuNk<| z7R#-qSLzXIU8kM6IMQk0q%;l)Eu$CiEDui%UT0MBL-Boli~(rkh+q1aklJD+j6YAb zbHC3A@NZM?dvluO`b4^d@;r@7wM%~j4THIZg+0v`HTa<*%zsu%- zXlLb(>CC*V(3I4aBrB_#)C~O?SW41dinc~VvK|tEIsQUW=svD$#|PaVkvSh1q%k%c zMZ-#h0XDGrc0c>x{5f--d#$ufXBH@>+ZrX8HDYoh!25i(TuWUKlCTBB95FN<=))B_ z_dAJ=*~IgG&V~CZuxW}9_~l=!=l1e6<0K~Bq|RrvzNm~vY)s*nUr*V=dPc2TB0m#D z{s|_R2#FOF&IAia%w|{SaCuLl-gFonkt=Q<>7bgwX3|2-0x0X#q9hvDnS~UE^fAec-qDAx*!}VG)6*A4;d4PHg!Q2SBw!#}lt1JV>Q6 zvWT=2BGPtyiNiWzs&ktHSjEZ6W{I_flxm%MLo7Zz0Lea+Y<+&}v3gbgA*qYV4G+r6 z6_1H0&neAtoTGuG>-`Fn6edGma6ekWy{!&7VTsug!uKhQtwrDX13O60ArpRHu-8;} zB2NUkEGJdrZb9B?HAzTQAo76L&wsb0R;XsiDl`)=B^w+C%wWYDfAhK{X_hFgS$n6s z4z_Pk#Ndl(LUO!}m)biPv=$=W=hoE%x>qB5<-P+g-s{k%!R53;MA(Hy;$)?sU7@QY z_|SgNXb_)dy}_Naqa!^L4Wc`cE&WPU!TH(o**Xrg%!kQ_U=Q}}^MV!gq|qM+it{)x zQ@hd;1cC0pMCKFO4pPYjAHbw(Dg)*z_|^z#BBkpF*}FCZ$&=vfDxvdqX@X;FHF7q- z%~HZNft_4=g#7|kx|d(7y$CjVQV{(B8eWS&XeO&r?R(bmkbUMnczyHzZt%^;8VXf# zMYh+}qr3NUUJB?3rSb{Bd`Ai^F)NQsxxc7%6fp@eDO}#$E<4;yPiF)q2c z@78q2+5KHZd*uO+TMsoV^$v;t-BgWyC5KO!=9)tV$9Ys)o|wd+l;%lw@qktJT%hi9 zLQtJ+9&&p2#>x&3LQwTT*=T{^s`FcM+GuuL4@!^=57|~*7e9DF+58eOT*xNHwcM+3 z{Lsew^`wsBBQPX>3~s|;-VJdRZt%#51u+E(-@%(2ssqlwkgeAh^CWH+m(-UL=%i5G zmTd8ET-cqOKb%7DLkn8l@ow>Q383pEbA1TN$Xq39J@KI*?GhcA9GglbROHEUuDC92 zgaeJl8K2eN6qC508AjK7waBR}3xmo?4zaux@tWk5^ZcA3;4(FjrOMzn*T{xom|A6N zyGvF1a2ZB$R17l>M?j>_#XdS5l8D#Kv)a4(wA2wucct)t!?RZ`rtHI1WRA${?6+5j ztX@1%Wk#n6kTM!xK)z{jlS7p?QHKs4ta#bXxhm0iq<080z(v`@g|FT3ouL__L}(00 znCO>vinRGsIs$DvR;I(=QVyp*49WGuAW7(1?rBZXcyO zVqZ_9!|#I~$ZSg_C4MgF6NeJSc+P!yD$tgrnuM=F_@a3~MF!#=zxuti?|egpr%O^3 z1^GPmk-6AFo zE7mI>)p{EB(QX1gNAZ^LcGL4r(BkwHp1c?&92#L2yhN--Kb{$5hp0v zZ!$yt^NEMy8kF&q?^koHyoSVp0b7z{48H4q%@7_H$f0nlYl+W#0qcGe>Q;l9YrBF_(BvDfzYm{(e z^jOw$A?;1u9Y6oo5a#8p8e zHW)pY=R=vYRh=r*S{ii{m<^7PK8-Zli$*I|bWgX*t*Q1_d72%6%GjcxnDRa#Ko?UP ziG>Mq>kp{v_oSpmK_-|}pMwvUN>1x+Q3D1y-ghDEJJ>CeJwN}dfa$&%}Knna-Q@rLdsUIXfq9Fy+7Q0H#LNjn6%Bp;XQXpA=) zIr%^LH|4I|%@Y>-)LIGG#A$Wyj50YEcWXh8*2=I=Tp0mBx=?ZtSj#{yBhC*=t*8x+kfRiL#SZ= zm`a1eiOVYAV^RxLV7|?ooiXf^ty|cBzN4s`W@>;1qF`X4$R_bt@iCoJ=ux6xGE#XF zRAnx#)~}V&LB!hhxP(!JRcZ}AhM>JTgpxPz#>6%40x{bdovrbXqW2qNo6{sCu{X~8 zYVCmF@{2e|hkIa2nkeEWDIUGKqT?jh)WgI*arr}{j0ys>A+ZF{v)60W?Gon!PBT0l z39bEN4bj~#TRD;OgbSV%emv)1>Iy#@d$F+6MSL%jy`1dfjY_K}q5cn7n$Xxs$-7dZ zPvBZRP>z8YMpi^-?7$mCGLNFH&J3~TUt&!X7fBWs(M7WHc9h`{=)hj_UN?@FaXQBY zmT7gud(A49@FAPZ_8Mt29?5Kr3_S0NGSh0hQ(T(i83h&wrWSV+bw@~*Z0qapSL?%m z(JUH35|Y}mj#sThp-(a6g@26DbdeP^E*IhVQqtt!v!$kN( zDe>izLn*I>D$@BL0`yIW&x>u<)aLbT3}o21J%A%tLe?vg zZb)b|Dc+6;>~;st?ALtm6J&p2(lbryIW-m=2I-$%FzN|K@6cV27 z?YFtft}Xw7k;r~7Qc7Wt7nB?c)aoSLiJjq?<)>yGazDb6E42*{i*8xB9nU{J}n0R{)7ocXm9qBVLa7CD;jIJ zuG_W8WH}hIW^(24#0^4-qqdvkde}Sr9h_tKcF*4zAGJ#TM2S;eO3_9sKHfB}y{13w zcs+tA3c5^H;MR1{=_%4sYc5p6Ga~{?_U~;1QjGoaG&8K3F~A)ZD<^is%>VbOR@T2@ zayVp^nxvza8Vmbac4FYt0IDkFK0Hp+8qSV~!jcti%r z#soz?oXm}!R1k_*K3*z*8YYBl)M^?9SA;8BNV1hn$vD(BNnvrx_aAjFv1-DLriRi} zk^)#VMGNa#8Zom4nM*lx_Id;7`r(-$+7LMoSvwW2$kcwO%y_?}BLWmb|J-%2Evi$f zDWRd$V|?FtPivM`sJBqM5X!P|o!tGcU}Ak$F#n-m{uAu9X!jND1Sb0y1cmtp>~9J0 zSBO8;?!F-s13pM5eMS6B$@}lo7yv(_js1Z3Q&sQJoSfG5zBxG|fgk+~3133~iId+b ze18_f3FY5@=T3d56l%gkEyJ%OIMI}T<>q_poK}>6{Lr*Nar1M1=~wQ*r^ac;n)ZGWP_{8u`E>{xJ$MR2TRy5ir=#2I*h9 v|Go~VH>)4rE06+yzjghYlhb|wM<=GJedB*kBo_8bo#3DRQ8`m~!YSZC-5Ux( literal 0 HcmV?d00001 From 5b831ec19de13753d7f649ffb78f3208bfd468d4 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 13 Jan 2025 13:50:18 +0200 Subject: [PATCH 17/41] update tests --- ...iveIndexIT.java => OldRepositoriesIT.java} | 20 +++--- .../oldrepos/SearchableSnapshotIT.java | 68 ------------------- 2 files changed, 10 insertions(+), 78 deletions(-) rename x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/{ArchiveIndexIT.java => OldRepositoriesIT.java} (89%) delete mode 100644 x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/SearchableSnapshotIT.java diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/ArchiveIndexIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java similarity index 89% rename from x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/ArchiveIndexIT.java rename to x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java index 1123fc2994c2a..067ae6db72845 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/ArchiveIndexIT.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java @@ -19,13 +19,13 @@ * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index * is successfully restored. */ -public class ArchiveIndexIT extends AbstractUpgradeCompatibilityTestCase { +public class OldRepositoriesIT extends AbstractUpgradeCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); } - public ArchiveIndexIT(Version version) { + public OldRepositoriesIT(Version version) { super(version); } @@ -36,7 +36,7 @@ public ArchiveIndexIT(Version version) { * 3. Index Restored to version: Current-1: 8.x * 4. Cluster upgraded to version Current: 9.x */ - public void testRestoreArchiveIndexVersion5() throws Exception { + public void testRestoreMountIndexVersion5() throws Exception { verifyCompatibility(TestSnapshotCases.ES_VERSION_5); } @@ -47,7 +47,7 @@ public void testRestoreArchiveIndexVersion5() throws Exception { * 3. Index Restored to version: Current-1: 8.x * 4. Cluster upgraded to version Current: 9.x */ - public void testMountSearchableSnapshotVersion6() throws Exception { + public void testRestoreMountIndexVersion6() throws Exception { verifyCompatibility(TestSnapshotCases.ES_VERSION_6); } @@ -83,7 +83,7 @@ public void testMountSearchableSnapshotVersion6() throws Exception { * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v5_standard_token_filter.zip) * 3. Index Restored to version: Current: 9.x */ - public void testRestoreVersion5StandardTokenFilter() throws Exception { + public void testRestoreMountVersion5StandardTokenFilter() throws Exception { verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> { assertEquals(1, warnings.size()); assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); @@ -122,7 +122,7 @@ public void testRestoreVersion5StandardTokenFilter() throws Exception { * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v6_standard_token_filter.zip) * 3. Index Restored to version: Current: 9.x */ - public void testRestoreVersion6StandardTokenFilter() throws Exception { + public void testRestoreMountVersion6StandardTokenFilter() throws Exception { verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> { assertEquals(1, warnings.size()); assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); @@ -136,7 +136,7 @@ public void testRestoreVersion6StandardTokenFilter() throws Exception { * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_80.zip) * 4. Index Restored to version: Current : 9.x */ - public void testRestoreVersion6LuceneCodec80() throws Exception { + public void testRestoreMountVersion6LuceneCodec80() throws Exception { verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_80); } @@ -147,7 +147,7 @@ public void testRestoreVersion6LuceneCodec80() throws Exception { * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_84.zip) * 4. Index Restored to version: Current: 9.x */ - public void testRestoreVersion6LuceneCodec84() throws Exception { + public void testRestoreMountVersion6LuceneCodec84() throws Exception { verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_84); } @@ -158,7 +158,7 @@ public void testRestoreVersion6LuceneCodec84() throws Exception { * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_86.zip) * 4. Index Restored to version: Current: 9.x */ - public void testRestoreVersion6LuceneCodec86() throws Exception { + public void testRestoreMountVersion6LuceneCodec86() throws Exception { verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_86); } @@ -169,7 +169,7 @@ public void testRestoreVersion6LuceneCodec86() throws Exception { * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_87.zip) * 4. Index Restored to version: Current: 9.x */ - public void testRestoreVersion6LuceneCodec87() throws Exception { + public void testRestoreMountVersion6LuceneCodec87() throws Exception { verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_87); } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/SearchableSnapshotIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/SearchableSnapshotIT.java deleted file mode 100644 index b4ff9fae8ad6f..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/SearchableSnapshotIT.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.WarningsHandler; -import org.elasticsearch.common.Strings; -import org.elasticsearch.test.cluster.util.Version; - -import java.util.List; -import java.util.function.Consumer; - -public class SearchableSnapshotIT extends AbstractUpgradeCompatibilityTestCase { - - static { - clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); - } - - public SearchableSnapshotIT(Version version) { - super(version); - } - - /** - * Test case restoring a snapshot created in ES_v6 - Basic mapping - * 1. Index Created in ES_v6 - * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v6.zip) - * 3. Index Restored to version: Current-1: 8.x - * 4. Cluster upgraded to version Current: 9.x - */ - public void testRestoreArchiveIndexVersion6() throws Exception { - verifyCompatibility(TestSnapshotCases.ES_VERSION_6); - } - - /** - * Test case mounting a snapshot created in ES_v5 - Basic mapping - * 1. Index Created in ES_v5 - * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v5.zip) - * 3. Index Restored to version: Current-1: 8.x - * 4. Cluster upgraded to version Current: 9.x - */ - public void testMountSearchableSnapshotVersion5() throws Exception { - verifyCompatibility(TestSnapshotCases.ES_VERSION_5); - } - - public void recover(RestClient client, String repository, String snapshot, String index, Consumer> warningsConsumer) - throws Exception { - var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount"); - request.addParameter("wait_for_completion", "true"); - request.addParameter("storage", "full_copy"); - request.setJsonEntity(Strings.format(""" - { - "index": "%s", - "renamed_index": "%s" - }""", index, index)); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - Response response = client.performRequest(request); - assertOK(response); - warningsConsumer.accept(response.getWarnings()); - } -} From 50530a23f347967d178d17d021d7cead9720452d Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 13 Jan 2025 15:07:52 +0200 Subject: [PATCH 18/41] Update library --- .../bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 7f164417160f6..d4026d2f4e552 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -29,7 +29,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.IOUtils; +import org.elasticsearch.core.IOUtils; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; import java.io.IOException; @@ -124,7 +124,7 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc success = true; } finally { if (success == false) { - org.apache.lucene.util.IOUtils.closeWhileHandlingException(this); + IOUtils.closeWhileHandlingException(this); } } } From 9be8ceac22ba523e96d30a47c4b0528aff4f3387 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 13 Jan 2025 15:25:17 +0200 Subject: [PATCH 19/41] update forbidden API --- .../xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index 0ea7b0a76846f..f1509c485cf89 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -46,11 +46,8 @@ public void testLuceneBWCCodecsAvailable() { Matcher classNameMatcher = classNamePattern.matcher(className); if (classNameMatcher.matches()) { String codecVersion = classNameMatcher.group(1); - String wrappedCodecClassPath = String.format( - "org.elasticsearch.xpack.lucene.bwc.codecs.lucene%s.BWCLucene%sCodec", - codecVersion, - codecVersion - ); + String wrappedCodecClassPath = """ + org.elasticsearch.xpack.lucene.bwc.codecs.lucene%s.BWCLucene%sCodec""".formatted(codecVersion, codecVersion); assertTrue(isClassPresent(wrappedCodecClassPath)); } } From 420e646d554e5f0d1b19815eb411b9f9954231b2 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 13 Jan 2025 15:57:55 +0200 Subject: [PATCH 20/41] Add smoke tests for codecs --- .../bwc/codecs/OldCodecsAvailableTests.java | 7 ++-- .../codecs/lucene80/Lucene80CodecTests.java | 32 +++++++++++++++++++ .../codecs/lucene84/Lucene84CodecTests.java | 32 +++++++++++++++++++ .../codecs/lucene86/Lucene86CodecTests.java | 32 +++++++++++++++++++ .../codecs/lucene87/Lucene87CodecTests.java | 32 +++++++++++++++++++ 5 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java create mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java create mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java create mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index f1509c485cf89..694f09ef1ea65 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -46,8 +46,11 @@ public void testLuceneBWCCodecsAvailable() { Matcher classNameMatcher = classNamePattern.matcher(className); if (classNameMatcher.matches()) { String codecVersion = classNameMatcher.group(1); - String wrappedCodecClassPath = """ - org.elasticsearch.xpack.lucene.bwc.codecs.lucene%s.BWCLucene%sCodec""".formatted(codecVersion, codecVersion); + String wrappedCodecClassPath = "org.elasticsearch.xpack.lucene.bwc.codecs.lucene" + + codecVersion + + "BWCLucene" + + codecVersion + + "Codec"; assertTrue(isClassPresent(wrappedCodecClassPath)); } } diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java new file mode 100644 index 0000000000000..46131bf10df1a --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80; + +import org.apache.lucene.codecs.Codec; +import org.elasticsearch.test.ESTestCase; + +public class Lucene80CodecTests extends ESTestCase { + + private final Codec codec; + + public Lucene80CodecTests() { + this.codec = new BWCLucene80Codec(); + } + + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java new file mode 100644 index 0000000000000..f7e52b751b458 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84; + +import org.apache.lucene.codecs.Codec; +import org.elasticsearch.test.ESTestCase; + +public class Lucene84CodecTests extends ESTestCase { + + private final Codec codec; + + public Lucene84CodecTests() { + this.codec = new BWCLucene84Codec(); + } + + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java new file mode 100644 index 0000000000000..240171d01c971 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.codecs.Codec; +import org.elasticsearch.test.ESTestCase; + +public class Lucene86CodecTests extends ESTestCase { + + private final Codec codec; + + public Lucene86CodecTests() { + this.codec = new BWCLucene86Codec(); + } + + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java new file mode 100644 index 0000000000000..8f3a7eec7afd7 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87; + +import org.apache.lucene.codecs.Codec; +import org.elasticsearch.test.ESTestCase; + +public class Lucene87CodecTests extends ESTestCase { + + private final Codec codec; + + public Lucene87CodecTests() { + this.codec = new BWCLucene87Codec(); + } + + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} From ad70e728e7ba1c5777ef79283f6b86e40fa60c7b Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 13 Jan 2025 16:27:04 +0200 Subject: [PATCH 21/41] fix test --- .../xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index 694f09ef1ea65..81a772904535e 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -48,7 +48,7 @@ public void testLuceneBWCCodecsAvailable() { String codecVersion = classNameMatcher.group(1); String wrappedCodecClassPath = "org.elasticsearch.xpack.lucene.bwc.codecs.lucene" + codecVersion - + "BWCLucene" + + ".BWCLucene" + codecVersion + "Codec"; assertTrue(isClassPresent(wrappedCodecClassPath)); From 99e816f7f87b3b3e126a00d9ccc3916e6c9b9b52 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 13 Jan 2025 16:55:09 +0200 Subject: [PATCH 22/41] Delete internal javadoc --- .../codecs/lucene86/Lucene86MetadataOnlyPointsReader.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index d4026d2f4e552..84b2cdd2f4cb3 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -129,11 +129,6 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc } } - /** - * Returns the underlying {@link PointValues}. - * - * @lucene.internal - */ @Override public PointValues getValues(String fieldName) { FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName); From 900fcb89891af8439e1373b4c8c77ada60cda6a3 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Wed, 15 Jan 2025 11:07:13 +0200 Subject: [PATCH 23/41] Modify MetadataOnly PointsReader to adjust Lucene86 changes --- .../lucene60/MetadataOnlyBKDReader.java | 14 +++++ .../Lucene86MetadataOnlyPointsReader.java | 57 ++++++++++--------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 43203caf571f1..24b82e23be9ef 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -47,6 +47,10 @@ public class MetadataOnlyBKDReader extends PointValues { final int docCount; final int version; + final int numIndexBytes; + final long minLeafBlockFP; + private final long indexStartPointer; + public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); final int numDims = metaIn.readVInt(); @@ -85,6 +89,16 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); + + numIndexBytes = metaIn.readVInt(); + if (version >= VERSION_META_FILE) { + minLeafBlockFP = metaIn.readLong(); + indexStartPointer = metaIn.readLong(); + } else { + indexStartPointer = metaIn.getFilePointer(); + minLeafBlockFP = metaIn.readVLong(); + metaIn.seek(indexStartPointer); + } } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 84b2cdd2f4cb3..4e769e171c034 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -46,25 +46,26 @@ public final class Lucene86MetadataOnlyPointsReader extends PointsReader { public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException { this.readState = readState; - String metaFileName = IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.META_EXTENSION - ); - String indexFileName = IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.INDEX_EXTENSION - ); - String dataFileName = IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.DATA_EXTENSION - ); + String metaFileName = + IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.META_EXTENSION); + String indexFileName = + IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.INDEX_EXTENSION); + String dataFileName = + IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.DATA_EXTENSION); boolean success = false; try { - indexIn = EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context); + indexIn = + EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context); CodecUtil.checkIndexHeader( indexIn, Lucene86MetadataOnlyPointsFormat.INDEX_CODEC_NAME, @@ -74,7 +75,8 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc readState.segmentSuffix ); - dataIn = EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context); + dataIn = + EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context); CodecUtil.checkIndexHeader( dataIn, Lucene86MetadataOnlyPointsFormat.DATA_CODEC_NAME, @@ -84,10 +86,10 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc readState.segmentSuffix ); - // long indexLength = -1, dataLength = -1; - try ( - ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context) - ) { + long indexLength = -1, dataLength = -1; + try (ChecksumIndexInput metaIn = + EndiannessReverserUtil.openChecksumInput( + readState.directory, metaFileName, readState.context)) { Throwable priorE = null; try { CodecUtil.checkIndexHeader( @@ -96,8 +98,7 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc Lucene86MetadataOnlyPointsFormat.VERSION_START, Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT, readState.segmentInfo.getId(), - readState.segmentSuffix - ); + readState.segmentSuffix); while (true) { int fieldNumber = metaIn.readInt(); @@ -109,18 +110,18 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc PointValues reader = new MetadataOnlyBKDReader(metaIn); readers.put(fieldNumber, reader); } - // indexLength = metaIn.readLong(); - // dataLength = metaIn.readLong(); + indexLength = metaIn.readLong(); + dataLength = metaIn.readLong(); } catch (Throwable t) { priorE = t; } finally { - // CodecUtil.checkFooter(metaIn, priorE); + CodecUtil.checkFooter(metaIn, priorE); } } // At this point, checksums of the meta file have been validated so we // know that indexLength and dataLength are very likely correct. - // CodecUtil.retrieveChecksum(indexIn, indexLength); - // CodecUtil.retrieveChecksum(dataIn, dataLength); + CodecUtil.retrieveChecksum(indexIn, indexLength); + CodecUtil.retrieveChecksum(dataIn, dataLength); success = true; } finally { if (success == false) { From 2abea6449935882c99da02d92f55e69de51b8462 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 15 Jan 2025 09:15:37 +0000 Subject: [PATCH 24/41] [CI] Auto commit changes from spotless --- .../Lucene86MetadataOnlyPointsReader.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 4e769e171c034..c587c4b55b57a 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -46,26 +46,25 @@ public final class Lucene86MetadataOnlyPointsReader extends PointsReader { public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException { this.readState = readState; - String metaFileName = - IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.META_EXTENSION); - String indexFileName = - IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.INDEX_EXTENSION); - String dataFileName = - IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.DATA_EXTENSION); + String metaFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.META_EXTENSION + ); + String indexFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.INDEX_EXTENSION + ); + String dataFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.DATA_EXTENSION + ); boolean success = false; try { - indexIn = - EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context); + indexIn = EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context); CodecUtil.checkIndexHeader( indexIn, Lucene86MetadataOnlyPointsFormat.INDEX_CODEC_NAME, @@ -75,8 +74,7 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc readState.segmentSuffix ); - dataIn = - EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context); + dataIn = EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context); CodecUtil.checkIndexHeader( dataIn, Lucene86MetadataOnlyPointsFormat.DATA_CODEC_NAME, @@ -87,9 +85,9 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc ); long indexLength = -1, dataLength = -1; - try (ChecksumIndexInput metaIn = - EndiannessReverserUtil.openChecksumInput( - readState.directory, metaFileName, readState.context)) { + try ( + ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context) + ) { Throwable priorE = null; try { CodecUtil.checkIndexHeader( @@ -98,7 +96,8 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc Lucene86MetadataOnlyPointsFormat.VERSION_START, Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT, readState.segmentInfo.getId(), - readState.segmentSuffix); + readState.segmentSuffix + ); while (true) { int fieldNumber = metaIn.readInt(); From 0609411d01e8aafd526d88bf2b3a7099eedacc0c Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 20 Jan 2025 16:24:07 +0200 Subject: [PATCH 25/41] update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 23 ++--- .../bwc/codecs/lucene70/BWCLucene70Codec.java | 42 ++++---- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 59 ++++++----- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 97 +++++++++++------- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 98 ++++++++++++------- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 98 ++++++++++++------- .../lucene/bwc/codecs/BWCCodecTests.java | 80 +++++++++++++++ .../codecs/lucene80/Lucene80CodecTests.java | 32 ------ .../codecs/lucene84/Lucene84CodecTests.java | 32 ------ .../codecs/lucene86/Lucene86CodecTests.java | 32 ------ .../codecs/lucene87/Lucene87CodecTests.java | 32 ------ 11 files changed, 320 insertions(+), 305 deletions(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index b5a75f5087446..3156ecd887189 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -7,10 +7,6 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; -import org.apache.lucene.backward_codecs.lucene80.Lucene80Codec; -import org.apache.lucene.backward_codecs.lucene84.Lucene84Codec; -import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec; -import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldsConsumer; @@ -160,16 +156,15 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { // Special handling for Lucene8xCodecs (which are currently bundled with Lucene) // Use BWCLucene8xCodec instead as that one extends BWCCodec (similar to all other older codecs) private static Codec getBackwardCompatibleCodec(Codec codec) { - if (codec instanceof Lucene80Codec) { - codec = new BWCLucene80Codec(); - } else if (codec instanceof Lucene84Codec) { - codec = new BWCLucene84Codec(); - } else if (codec instanceof Lucene86Codec) { - codec = new BWCLucene86Codec(); - } else if (codec instanceof Lucene87Codec) { - codec = new BWCLucene87Codec(); - } - return codec; + if (codec == null) return null; + + return switch (codec.getClass().getSimpleName()) { + case "Lucene80Codec" -> new BWCLucene80Codec(); + case "Lucene84Codec" -> new BWCLucene84Codec(); + case "Lucene86Codec" -> new BWCLucene86Codec(); + case "Lucene87Codec" -> new BWCLucene87Codec(); + default -> codec; + }; } /** diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index f3d80cbcfe842..9eabf2203e8ba 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -12,7 +12,6 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; -import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; @@ -26,20 +25,26 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; /** - * Implements the Lucene 7.0 index format. Loaded via SPI for indices created/written with Lucene 7.x (Elasticsearch 6.x) mounted - * as archive indices first in Elasticsearch 8.x. Lucene 9.12 retained Lucene70Codec in its classpath which required overriding the - * codec name and version in the segment infos. This codec is still needed after upgrading to Elasticsearch 9.x because its codec - * name has been written to disk. - */ +* Implements the Lucene 7.0 index format. Loaded via SPI for indices created/written with Lucene 7.x (Elasticsearch 6.x) mounted +* as archive indices first in Elasticsearch 8.x. Lucene 9.12 retained Lucene70Codec in its classpath which required overriding the +* codec name and version in the segment infos. This codec is still needed after upgrading to Elasticsearch 9.x because its codec +* name has been written to disk. +*/ public class BWCLucene70Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat; - private final SegmentInfoFormat segmentInfosFormat; + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; - private final LiveDocsFormat liveDocsFormat; - private final CompoundFormat compoundFormat; - private final DocValuesFormat docValuesFormat; - private final PointsFormat pointsFormat; + private final DocValuesFormat defaultDVFormat = new Lucene70DocValuesFormat(); + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); // Needed for SPI loading @SuppressWarnings("unused") @@ -49,18 +54,7 @@ public BWCLucene70Codec() { protected BWCLucene70Codec(String name) { super(name); - this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - this.segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); - this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - this.liveDocsFormat = new Lucene50LiveDocsFormat(); - this.compoundFormat = new Lucene50CompoundFormat(); - this.docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return new Lucene80DocValuesFormat(); - } - }; - this.pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index 04e33cc983ed7..b6e98deef4d74 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -31,13 +31,21 @@ */ public class BWCLucene80Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat; - private final SegmentInfoFormat segmentInfosFormat; + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); + private final StoredFieldsFormat storedFieldsFormat; - private final LiveDocsFormat liveDocsFormat; - private final CompoundFormat compoundFormat; - private final DocValuesFormat docValuesFormat; - private final PointsFormat pointsFormat; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); // Needed for SPI loading @SuppressWarnings("unused") @@ -47,18 +55,17 @@ public BWCLucene80Codec() { public BWCLucene80Codec(String name) { super(name); - this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - this.segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - this.liveDocsFormat = new Lucene50LiveDocsFormat(); - this.compoundFormat = new Lucene50CompoundFormat(); - this.docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return new Lucene80DocValuesFormat(); - } - }; - this.pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final PostingsFormat postingsFormat() { + return postingsFormat; } @Override @@ -71,11 +78,6 @@ public final SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } - @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -87,17 +89,12 @@ public final CompoundFormat compoundFormat() { } @Override - public final DocValuesFormat docValuesFormat() { - return docValuesFormat; - } - - @Override - public final PostingsFormat postingsFormat() { - return postingsFormat; + public final PointsFormat pointsFormat() { + return pointsFormat; } @Override - public final PointsFormat pointsFormat() { - return pointsFormat; + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 41f41503a13bb..550f4f40655d5 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -33,14 +33,28 @@ */ public class BWCLucene84Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat; - private final SegmentInfoFormat segmentInfosFormat; + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene84Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene84Codec.this.getDocValuesFormatForField(field); + } + }; + private final StoredFieldsFormat storedFieldsFormat; - private final LiveDocsFormat liveDocsFormat; - private final CompoundFormat compoundFormat; - private final DocValuesFormat docValuesFormat; - private final PostingsFormat postingsFormat; - private final PointsFormat pointsFormat; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); // Needed for SPI loading @SuppressWarnings("unused") @@ -50,24 +64,18 @@ public BWCLucene84Codec() { public BWCLucene84Codec(String name) { super(name); - this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - this.segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - this.liveDocsFormat = new Lucene50LiveDocsFormat(); - this.compoundFormat = new Lucene50CompoundFormat(); - this.docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return new Lucene80DocValuesFormat(); - } - }; - this.postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return new Lucene84PostingsFormat(); - } - }; - this.pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + this.defaultFormat = new Lucene84PostingsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; } @Override @@ -80,11 +88,6 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -96,17 +99,39 @@ public CompoundFormat compoundFormat() { } @Override - public final DocValuesFormat docValuesFormat() { - return docValuesFormat; + public PointsFormat pointsFormat() { + return pointsFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; } @Override - public PointsFormat pointsFormat() { - return pointsFormat; + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; } + + private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index b4b9eb2e84dc0..14f199e5d255d 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -32,14 +32,28 @@ */ public class BWCLucene86Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat; - private final SegmentInfoFormat segmentInfosFormat; + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene86Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene86Codec.this.getDocValuesFormatForField(field); + } + }; + private final StoredFieldsFormat storedFieldsFormat; - private final LiveDocsFormat liveDocsFormat; - private final CompoundFormat compoundFormat; - private final DocValuesFormat docValuesFormat; - private final PostingsFormat postingsFormat; - private final PointsFormat pointsFormat; // Needed for SPI loading @SuppressWarnings("unused") @@ -50,24 +64,18 @@ public BWCLucene86Codec() { /** Instantiates a new codec. */ public BWCLucene86Codec(String name) { super(name); - this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - this.segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - this.liveDocsFormat = new Lucene50LiveDocsFormat(); - this.compoundFormat = new Lucene50CompoundFormat(); - this.docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return new Lucene80DocValuesFormat(); - } - }; - this.postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return new Lucene84PostingsFormat(); - } - }; - this.pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + this.defaultFormat = new Lucene84PostingsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; } @Override @@ -80,11 +88,6 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -96,17 +99,40 @@ public CompoundFormat compoundFormat() { } @Override - public final DocValuesFormat docValuesFormat() { - return docValuesFormat; + public PointsFormat pointsFormat() { + return pointsFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; } @Override - public PointsFormat pointsFormat() { - return pointsFormat; + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; } + + private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); + } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index 627c013d2dceb..5173f142e3505 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -53,14 +53,28 @@ private enum Mode { } } - private final FieldInfosFormat fieldInfosFormat; - private final SegmentInfoFormat segmentInfosFormat; + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene87Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene87Codec.this.getDocValuesFormatForField(field); + } + }; + private final StoredFieldsFormat storedFieldsFormat; - private final LiveDocsFormat liveDocsFormat; - private final CompoundFormat compoundFormat; - private final DocValuesFormat docValuesFormat; - private final PostingsFormat postingsFormat; - private final PointsFormat pointsFormat; // Needed for SPI loading @SuppressWarnings("unused") @@ -70,24 +84,19 @@ public BWCLucene87Codec() { public BWCLucene87Codec(String name, Mode mode) { super(name); - this.fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - this.segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); this.storedFieldsFormat = new Lucene87StoredFieldsFormat(mode.storedMode); - this.liveDocsFormat = new Lucene50LiveDocsFormat(); - this.compoundFormat = new Lucene50CompoundFormat(); - this.docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return new Lucene80DocValuesFormat(mode.dvMode); - } - }; - this.postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return new Lucene84PostingsFormat(); - } - }; - this.pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + this.defaultFormat = new Lucene84PostingsFormat(); + this.defaultDVFormat = new Lucene80DocValuesFormat(mode.dvMode); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; } @Override @@ -100,11 +109,6 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; @@ -116,17 +120,39 @@ public CompoundFormat compoundFormat() { } @Override - public final DocValuesFormat docValuesFormat() { - return docValuesFormat; + public PointsFormat pointsFormat() { + return pointsFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; } @Override - public PointsFormat pointsFormat() { - return pointsFormat; + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; } + + private final DocValuesFormat defaultDVFormat; } diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java new file mode 100644 index 0000000000000..1c0739e40fee1 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.elasticsearch.test.ESTestCase; + +public class BWCCodecTests extends ESTestCase { + + private final Codec codec; + + public BWCCodecTests() { + this.codec = new BWCCodec("WrapperCodec") { + @Override + public PostingsFormat postingsFormat() { + return null; + } + + @Override + public DocValuesFormat docValuesFormat() { + return null; + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return null; + } + + @Override + public FieldInfosFormat fieldInfosFormat() { + return null; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return null; + } + + @Override + public LiveDocsFormat liveDocsFormat() { + return null; + } + + @Override + public CompoundFormat compoundFormat() { + return null; + } + + @Override + public PointsFormat pointsFormat() { + return null; + } + }; + } + + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java deleted file mode 100644 index 46131bf10df1a..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/Lucene80CodecTests.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80; - -import org.apache.lucene.codecs.Codec; -import org.elasticsearch.test.ESTestCase; - -public class Lucene80CodecTests extends ESTestCase { - - private final Codec codec; - - public Lucene80CodecTests() { - this.codec = new BWCLucene80Codec(); - } - - public void testNormsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::normsFormat); - } - - public void testTermVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); - } - - public void testKnnVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java deleted file mode 100644 index f7e52b751b458..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/Lucene84CodecTests.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84; - -import org.apache.lucene.codecs.Codec; -import org.elasticsearch.test.ESTestCase; - -public class Lucene84CodecTests extends ESTestCase { - - private final Codec codec; - - public Lucene84CodecTests() { - this.codec = new BWCLucene84Codec(); - } - - public void testNormsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::normsFormat); - } - - public void testTermVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); - } - - public void testKnnVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java deleted file mode 100644 index 240171d01c971..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86CodecTests.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; - -import org.apache.lucene.codecs.Codec; -import org.elasticsearch.test.ESTestCase; - -public class Lucene86CodecTests extends ESTestCase { - - private final Codec codec; - - public Lucene86CodecTests() { - this.codec = new BWCLucene86Codec(); - } - - public void testNormsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::normsFormat); - } - - public void testTermVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); - } - - public void testKnnVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java deleted file mode 100644 index 8f3a7eec7afd7..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/Lucene87CodecTests.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87; - -import org.apache.lucene.codecs.Codec; -import org.elasticsearch.test.ESTestCase; - -public class Lucene87CodecTests extends ESTestCase { - - private final Codec codec; - - public Lucene87CodecTests() { - this.codec = new BWCLucene87Codec(); - } - - public void testNormsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::normsFormat); - } - - public void testTermVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); - } - - public void testKnnVectorsFormatUnsupportedOperation() { - assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); - } -} From d9dedc7e269d424984100db36639b225ca3b0434 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 20 Jan 2025 16:25:56 +0200 Subject: [PATCH 26/41] revert code after review --- .../lucene/bwc/codecs/lucene70/BWCLucene70Codec.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index 9eabf2203e8ba..b5d8ea9178b29 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -25,11 +25,11 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; /** -* Implements the Lucene 7.0 index format. Loaded via SPI for indices created/written with Lucene 7.x (Elasticsearch 6.x) mounted -* as archive indices first in Elasticsearch 8.x. Lucene 9.12 retained Lucene70Codec in its classpath which required overriding the -* codec name and version in the segment infos. This codec is still needed after upgrading to Elasticsearch 9.x because its codec -* name has been written to disk. -*/ + * Implements the Lucene 7.0 index format. Loaded via SPI for indices created/written with Lucene 7.x (Elasticsearch 6.x) mounted + * as archive indices first in Elasticsearch 8.x. Lucene 9.12 retained Lucene70Codec in its classpath which required overriding the + * codec name and version in the segment infos. This codec is still needed after upgrading to Elasticsearch 9.x because its codec + * name has been written to disk. + */ public class BWCLucene70Codec extends BWCCodec { private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); From de31af333470a55aab9e4bc95d22dcfff098ad6a Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Wed, 22 Jan 2025 12:01:54 +0200 Subject: [PATCH 27/41] update after review --- docs/changelog/119503.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/119503.yaml b/docs/changelog/119503.yaml index eb4d641906c5b..da6697eea7b3a 100644 --- a/docs/changelog/119503.yaml +++ b/docs/changelog/119503.yaml @@ -1,5 +1,5 @@ pr: 119503 -summary: Support 7x segments as archive in 8x / 9x +summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. area: Search type: bug issues: From 2ca8a9ff5221d1c32a87395b71fd19367dff84b4 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 24 Jan 2025 12:31:20 +0200 Subject: [PATCH 28/41] Update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 25 ++++++++++++++----- .../bwc/codecs/lucene60/Lucene60Codec.java | 13 ++++++++-- .../bwc/codecs/lucene62/Lucene62Codec.java | 13 ++++++++-- .../bwc/codecs/lucene70/BWCLucene70Codec.java | 12 +++++++-- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 12 +++++++-- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 12 +++++++-- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 12 +++++++-- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 12 +++++++-- .../lucene/bwc/codecs/BWCCodecTests.java | 10 ++++++++ 9 files changed, 101 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 3156ecd887189..bb9a75640e922 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -43,8 +43,13 @@ */ public abstract class BWCCodec extends Codec { + protected FieldInfosFormat fieldInfosFormat; + protected SegmentInfoFormat segmentInfosFormat; + protected BWCCodec(String name) { super(name); + this.fieldInfosFormat = wrappedFieldInfosFormat(); + this.segmentInfosFormat = wrappedSegmentInfoFormat(); } protected final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @@ -69,32 +74,40 @@ public final KnnVectorsFormat knnVectorsFormat() { throw new UnsupportedOperationException(); } - protected static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) { + protected abstract SegmentInfoFormat setSegmentInfoFormat(); + + protected final SegmentInfoFormat wrappedSegmentInfoFormat() { return new SegmentInfoFormat() { + final SegmentInfoFormat wrappedFormat = setSegmentInfoFormat(); + @Override public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { - return wrap(wrapped.read(directory, segmentName, segmentID, context)); + return wrap(wrappedFormat.read(directory, segmentName, segmentID, context)); } @Override public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException { - wrapped.write(dir, info, ioContext); + wrappedFormat.write(dir, info, ioContext); } }; } - protected static FieldInfosFormat wrap(FieldInfosFormat wrapped) { + protected abstract FieldInfosFormat setFieldInfosFormat(); + + protected final FieldInfosFormat wrappedFieldInfosFormat() { return new FieldInfosFormat() { + final FieldInfosFormat wrappedFormat = setFieldInfosFormat(); + @Override public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException { - return filterFields(wrapped.read(directory, segmentInfo, segmentSuffix, iocontext)); + return filterFields(wrappedFormat.read(directory, segmentInfo, segmentSuffix, iocontext)); } @Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { - wrapped.write(directory, segmentInfo, segmentSuffix, infos, context); + wrappedFormat.write(directory, segmentInfo, segmentSuffix, infos, context); } }; } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java index 9694c8bf34d67..109f566dd6707 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java @@ -47,8 +47,17 @@ */ @Deprecated public class Lucene60Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene50SegmentInfoFormat()); + + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene50SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java index 61579d33e41cb..ccbfc56b0f657 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java @@ -47,8 +47,17 @@ */ @Deprecated public class Lucene62Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene62SegmentInfoFormat()); + + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene62SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index b5d8ea9178b29..742192e92ddc0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -32,8 +32,16 @@ */ public class BWCLucene70Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index b6e98deef4d74..b41266a57f2b5 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -31,8 +31,16 @@ */ public class BWCLucene80Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 550f4f40655d5..0930dd2d389b1 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -33,8 +33,16 @@ */ public class BWCLucene84Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PostingsFormat defaultFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 14f199e5d255d..69f87555041a1 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -32,8 +32,16 @@ */ public class BWCLucene86Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index 5173f142e3505..ddce0fdf0512e 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -53,8 +53,16 @@ private enum Mode { } } - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); + } + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java index 1c0739e40fee1..8ff04df8a9fcc 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java @@ -24,6 +24,16 @@ public class BWCCodecTests extends ESTestCase { public BWCCodecTests() { this.codec = new BWCCodec("WrapperCodec") { + @Override + protected SegmentInfoFormat setSegmentInfoFormat() { + return null; + } + + @Override + protected FieldInfosFormat setFieldInfosFormat() { + return null; + } + @Override public PostingsFormat postingsFormat() { return null; From 3e0de6d847cc37f8e311c0ce3ce04b583c93bf8a Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 27 Jan 2025 12:43:38 +0200 Subject: [PATCH 29/41] Update after review --- .../lucene60/MetadataOnlyBKDReader.java | 11 ++--- .../Lucene86MetadataOnlyPointsFormat.java | 8 ---- .../Lucene86MetadataOnlyPointsReader.java | 44 ++----------------- 3 files changed, 6 insertions(+), 57 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 24b82e23be9ef..f4e484b80d068 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -90,15 +90,10 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); + // This code has been introduced to process IndexInput created with Lucene86Codec+ numIndexBytes = metaIn.readVInt(); - if (version >= VERSION_META_FILE) { - minLeafBlockFP = metaIn.readLong(); - indexStartPointer = metaIn.readLong(); - } else { - indexStartPointer = metaIn.getFilePointer(); - minLeafBlockFP = metaIn.readVLong(); - metaIn.seek(indexStartPointer); - } + minLeafBlockFP = metaIn.readLong(); + indexStartPointer = metaIn.readLong(); } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java index 2dbd254d592ba..881061273f7e9 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java @@ -32,16 +32,8 @@ **/ public class Lucene86MetadataOnlyPointsFormat extends PointsFormat { - static final String DATA_CODEC_NAME = "Lucene86PointsFormatData"; - static final String INDEX_CODEC_NAME = "Lucene86PointsFormatIndex"; static final String META_CODEC_NAME = "Lucene86PointsFormatMeta"; - /** Filename extension for the leaf blocks */ - public static final String DATA_EXTENSION = "kdd"; - - /** Filename extension for the index per field */ - public static final String INDEX_EXTENSION = "kdi"; - /** Filename extension for the meta per field */ public static final String META_EXTENSION = "kdm"; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index c587c4b55b57a..699608fd541db 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -28,7 +28,6 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.IndexInput; import org.elasticsearch.core.IOUtils; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; @@ -38,11 +37,9 @@ /** Reads the metadata of point values previously written with Lucene86PointsWriter */ public final class Lucene86MetadataOnlyPointsReader extends PointsReader { - final IndexInput indexIn, dataIn; final SegmentReadState readState; final Map readers = new HashMap<>(); - /** Sole constructor */ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException { this.readState = readState; @@ -51,39 +48,9 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc readState.segmentSuffix, Lucene86MetadataOnlyPointsFormat.META_EXTENSION ); - String indexFileName = IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.INDEX_EXTENSION - ); - String dataFileName = IndexFileNames.segmentFileName( - readState.segmentInfo.name, - readState.segmentSuffix, - Lucene86MetadataOnlyPointsFormat.DATA_EXTENSION - ); boolean success = false; try { - indexIn = EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context); - CodecUtil.checkIndexHeader( - indexIn, - Lucene86MetadataOnlyPointsFormat.INDEX_CODEC_NAME, - Lucene86MetadataOnlyPointsFormat.VERSION_START, - Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT, - readState.segmentInfo.getId(), - readState.segmentSuffix - ); - - dataIn = EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context); - CodecUtil.checkIndexHeader( - dataIn, - Lucene86MetadataOnlyPointsFormat.DATA_CODEC_NAME, - Lucene86MetadataOnlyPointsFormat.VERSION_START, - Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT, - readState.segmentInfo.getId(), - readState.segmentSuffix - ); - long indexLength = -1, dataLength = -1; try ( ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context) @@ -117,10 +84,7 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc CodecUtil.checkFooter(metaIn, priorE); } } - // At this point, checksums of the meta file have been validated so we - // know that indexLength and dataLength are very likely correct. - CodecUtil.retrieveChecksum(indexIn, indexLength); - CodecUtil.retrieveChecksum(dataIn, dataLength); + success = true; } finally { if (success == false) { @@ -143,14 +107,12 @@ public PointValues getValues(String fieldName) { } @Override - public void checkIntegrity() throws IOException { - CodecUtil.checksumEntireFile(indexIn); - CodecUtil.checksumEntireFile(dataIn); + public void checkIntegrity() { + ; } @Override public void close() throws IOException { - IOUtils.close(indexIn, dataIn); // Free up heap: readers.clear(); } From bb561e2bf6ead6cfe7154523502fd99c5b7a37b6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 27 Jan 2025 10:51:37 +0000 Subject: [PATCH 30/41] [CI] Auto commit changes from spotless --- .../bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 699608fd541db..232aa6ba949d9 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -108,7 +108,7 @@ public PointValues getValues(String fieldName) { @Override public void checkIntegrity() { - ; + ; } @Override From 9c9dc665a1fe099503efd24320eaa239402bf0c4 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 30 Jan 2025 10:28:01 +0200 Subject: [PATCH 31/41] Update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 84 ++++++++++--------- .../bwc/codecs/lucene60/Lucene60Codec.java | 22 ++--- .../bwc/codecs/lucene62/Lucene62Codec.java | 22 ++--- .../bwc/codecs/lucene70/BWCLucene70Codec.java | 19 ++--- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 26 ++---- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 26 ++---- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 26 ++---- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 26 ++---- .../lucene/bwc/codecs/BWCCodecTests.java | 14 +--- 9 files changed, 95 insertions(+), 170 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index bb9a75640e922..aebca9d045a70 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -43,13 +43,41 @@ */ public abstract class BWCCodec extends Codec { - protected FieldInfosFormat fieldInfosFormat; - protected SegmentInfoFormat segmentInfosFormat; + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; protected BWCCodec(String name) { super(name); - this.fieldInfosFormat = wrappedFieldInfosFormat(); - this.segmentInfosFormat = wrappedSegmentInfoFormat(); + + this.fieldInfosFormat = new FieldInfosFormat() { + final FieldInfosFormat wrappedFormat = originalFieldInfosFormat(); + + @Override + public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) + throws IOException { + return filterFields(wrappedFormat.read(directory, segmentInfo, segmentSuffix, iocontext)); + } + + @Override + public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) + throws IOException { + wrappedFormat.write(directory, segmentInfo, segmentSuffix, infos, context); + } + }; + + this.segmentInfosFormat = new SegmentInfoFormat() { + final SegmentInfoFormat wrappedFormat = originalSegmentInfoFormat(); + + @Override + public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { + return wrap(wrappedFormat.read(directory, segmentName, segmentID, context)); + } + + @Override + public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException { + wrappedFormat.write(dir, info, ioContext); + } + }; } protected final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @@ -59,6 +87,16 @@ public PostingsFormat getPostingsFormatForField(String field) { } }; + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + @Override public final NormsFormat normsFormat() { throw new UnsupportedOperationException(); @@ -74,43 +112,9 @@ public final KnnVectorsFormat knnVectorsFormat() { throw new UnsupportedOperationException(); } - protected abstract SegmentInfoFormat setSegmentInfoFormat(); - - protected final SegmentInfoFormat wrappedSegmentInfoFormat() { - return new SegmentInfoFormat() { - final SegmentInfoFormat wrappedFormat = setSegmentInfoFormat(); - - @Override - public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { - return wrap(wrappedFormat.read(directory, segmentName, segmentID, context)); - } - - @Override - public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException { - wrappedFormat.write(dir, info, ioContext); - } - }; - } - - protected abstract FieldInfosFormat setFieldInfosFormat(); - - protected final FieldInfosFormat wrappedFieldInfosFormat() { - return new FieldInfosFormat() { - final FieldInfosFormat wrappedFormat = setFieldInfosFormat(); - - @Override - public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) - throws IOException { - return filterFields(wrappedFormat.read(directory, segmentInfo, segmentSuffix, iocontext)); - } + protected abstract SegmentInfoFormat originalSegmentInfoFormat(); - @Override - public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) - throws IOException { - wrappedFormat.write(directory, segmentInfo, segmentSuffix, infos, context); - } - }; - } + protected abstract FieldInfosFormat originalFieldInfosFormat(); // mark all fields as no term vectors, no norms, no payloads, and no vectors. private static FieldInfos filterFields(FieldInfos fieldInfos) { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java index 109f566dd6707..9ff5cace8ecd0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java @@ -48,16 +48,6 @@ @Deprecated public class Lucene60Codec extends BWCCodec { - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene50SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -98,18 +88,18 @@ public Lucene60Codec(Lucene50StoredFieldsFormat.Mode mode) { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene50SegmentInfoFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java index ccbfc56b0f657..adc1f97c71f12 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java @@ -48,16 +48,6 @@ @Deprecated public class Lucene62Codec extends BWCCodec { - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene62SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -89,18 +79,18 @@ public Lucene62Codec(Lucene50StoredFieldsFormat.Mode mode) { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene62SegmentInfoFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index 742192e92ddc0..5dc3dfa213315 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -32,16 +32,6 @@ */ public class BWCLucene70Codec extends BWCCodec { - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene70SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -66,15 +56,16 @@ protected BWCLucene70Codec(String name) { } @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); } + @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index b41266a57f2b5..5abeb19aa7a07 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -31,16 +31,6 @@ */ public class BWCLucene80Codec extends BWCCodec { - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene70SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); @@ -67,23 +57,23 @@ public BWCLucene80Codec(String name) { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final PostingsFormat postingsFormat() { - return postingsFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override - public final SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final PostingsFormat postingsFormat() { + return postingsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 0930dd2d389b1..4c321333319a3 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -33,16 +33,6 @@ */ public class BWCLucene84Codec extends BWCCodec { - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene70SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PostingsFormat defaultFormat; @@ -77,23 +67,23 @@ public BWCLucene84Codec(String name) { } @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public PostingsFormat postingsFormat() { + return postingsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 69f87555041a1..9e479654c1ad5 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -32,16 +32,6 @@ */ public class BWCLucene86Codec extends BWCCodec { - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene86SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); @@ -77,23 +67,23 @@ public BWCLucene86Codec(String name) { } @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public PostingsFormat postingsFormat() { + return postingsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index ddce0fdf0512e..ce148047f4774 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -53,16 +53,6 @@ private enum Mode { } } - @Override - protected FieldInfosFormat setFieldInfosFormat() { - return new Lucene60FieldInfosFormat(); - } - - @Override - protected SegmentInfoFormat setSegmentInfoFormat() { - return new Lucene86SegmentInfoFormat(); - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); @@ -98,23 +88,23 @@ public BWCLucene87Codec(String name, Mode mode) { } @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public PostingsFormat postingsFormat() { + return postingsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java index 8ff04df8a9fcc..ac50db344c38b 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java @@ -25,12 +25,12 @@ public class BWCCodecTests extends ESTestCase { public BWCCodecTests() { this.codec = new BWCCodec("WrapperCodec") { @Override - protected SegmentInfoFormat setSegmentInfoFormat() { + protected SegmentInfoFormat originalSegmentInfoFormat() { return null; } @Override - protected FieldInfosFormat setFieldInfosFormat() { + protected FieldInfosFormat originalFieldInfosFormat() { return null; } @@ -49,16 +49,6 @@ public StoredFieldsFormat storedFieldsFormat() { return null; } - @Override - public FieldInfosFormat fieldInfosFormat() { - return null; - } - - @Override - public SegmentInfoFormat segmentInfoFormat() { - return null; - } - @Override public LiveDocsFormat liveDocsFormat() { return null; From 6e316239e16f59b96eb7b62ea01e5fd638fe08b5 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 30 Jan 2025 12:47:24 +0200 Subject: [PATCH 32/41] update after review --- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 33 +++--------------- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 34 +++---------------- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 32 ++--------------- 3 files changed, 11 insertions(+), 88 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 4c321333319a3..1957221256041 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -36,18 +36,19 @@ public class BWCLucene84Codec extends BWCCodec { private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PostingsFormat defaultFormat; + private final DocValuesFormat defaultDVFormat; private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene84Codec.this.getPostingsFormatForField(field); + return defaultFormat; } }; private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene84Codec.this.getDocValuesFormatForField(field); + return defaultDVFormat; } }; @@ -64,6 +65,7 @@ public BWCLucene84Codec(String name) { super(name); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); this.defaultFormat = new Lucene84PostingsFormat(); + this.defaultDVFormat = new Lucene80DocValuesFormat(); } @Override @@ -101,35 +103,8 @@ public PointsFormat pointsFormat() { return pointsFormat; } - /** - * Returns the postings format that should be used for writing new segments of field. - * - *

The default implementation always returns "Lucene84". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - - /** - * Returns the docvalues format that should be used for writing new segments of field - * . - * - *

The default implementation always returns "Lucene80". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - - private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 9e479654c1ad5..4a02f11683798 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -36,18 +36,19 @@ public class BWCLucene86Codec extends BWCCodec { private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); private final PostingsFormat defaultFormat; + private final DocValuesFormat defaultDVFormat; private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene86Codec.this.getPostingsFormatForField(field); + return defaultFormat; } }; private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene86Codec.this.getDocValuesFormatForField(field); + return defaultDVFormat; } }; @@ -64,6 +65,7 @@ public BWCLucene86Codec(String name) { super(name); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); this.defaultFormat = new Lucene84PostingsFormat(); + this.defaultDVFormat = new Lucene80DocValuesFormat(); } @Override @@ -101,36 +103,8 @@ public PointsFormat pointsFormat() { return pointsFormat; } - /** - * Returns the postings format that should be used for writing new segments of field. - * - *

The default implementation always returns "Lucene84". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - - /** - * Returns the docvalues format that should be used for writing new segments of field - * . - * - *

The default implementation always returns "Lucene80". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - - private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); - } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index ce148047f4774..57bb515a3a6e8 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -57,18 +57,19 @@ private enum Mode { private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); private final PostingsFormat defaultFormat; + private final DocValuesFormat defaultDVFormat; private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override public PostingsFormat getPostingsFormatForField(String field) { - return BWCLucene87Codec.this.getPostingsFormatForField(field); + return defaultFormat; } }; private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { - return BWCLucene87Codec.this.getDocValuesFormatForField(field); + return defaultDVFormat; } }; @@ -122,35 +123,8 @@ public PointsFormat pointsFormat() { return pointsFormat; } - /** - * Returns the postings format that should be used for writing new segments of field. - * - *

The default implementation always returns "Lucene84". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - - /** - * Returns the docvalues format that should be used for writing new segments of field - * . - * - *

The default implementation always returns "Lucene80". - * - *

WARNING: if you subclass, you are responsible for index backwards compatibility: - * future version of Lucene are only guaranteed to be able to read the default implementation. - */ - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - @Override public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - - private final DocValuesFormat defaultDVFormat; } From f155836e24436ee013973cd2aeacdafcf1b36fdb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 30 Jan 2025 10:53:29 +0000 Subject: [PATCH 33/41] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java | 2 +- .../xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index aebca9d045a70..6e72975bdb241 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -49,7 +49,7 @@ public abstract class BWCCodec extends Codec { protected BWCCodec(String name) { super(name); - this.fieldInfosFormat = new FieldInfosFormat() { + this.fieldInfosFormat = new FieldInfosFormat() { final FieldInfosFormat wrappedFormat = originalFieldInfosFormat(); @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index 5dc3dfa213315..f9b70d5a0e0ef 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -65,7 +65,6 @@ protected SegmentInfoFormat originalSegmentInfoFormat() { return new Lucene70SegmentInfoFormat(); } - @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; From 153984439a9088c6e8fd7f470b7e90317ef7b357 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Mon, 10 Mar 2025 12:42:27 +0200 Subject: [PATCH 34/41] update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 46 ++++++++++++++++++- .../lucene60/MetadataOnlyBKDReader.java | 6 +-- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 7 +-- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 7 +-- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 19 +------- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 31 ++----------- .../lucene/bwc/codecs/BWCCodecTests.java | 12 +++++ .../bwc/codecs/OldCodecsAvailableTests.java | 2 +- 8 files changed, 70 insertions(+), 60 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 6e72975bdb241..250c963520b81 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -28,6 +28,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Version; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; @@ -97,23 +98,60 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * This method is never called in practice, as we rewrite field infos to override the info about which features are present in + * the index. Even if norms are present, field info lies about it. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ @Override public final NormsFormat normsFormat() { throw new UnsupportedOperationException(); } + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * This method is never called in practice, as we rewrite field infos to override the info about which features are present in + * the index. Even if term vectors are present, field info lies about it. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ @Override public final TermVectorsFormat termVectorsFormat() { throw new UnsupportedOperationException(); } + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * The knn vectors can't be present because it is not supported yet in any of the lucene versions that we support for archive indices. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ @Override public final KnnVectorsFormat knnVectorsFormat() { throw new UnsupportedOperationException(); } + /** + * Returns the original {@link SegmentInfoFormat} used by this codec. + * This method should be implemented by subclasses to provide the specific + * {@link SegmentInfoFormat} that this codec is intended to use. + * + * @return the original {@link SegmentInfoFormat} used by this codec + */ protected abstract SegmentInfoFormat originalSegmentInfoFormat(); + /** + * Returns the original {@link FieldInfosFormat} used by this codec. + * This method should be implemented by subclasses to provide the specific + * {@link FieldInfosFormat} that this codec is intended to use. + * + * @return the original {@link FieldInfosFormat} used by this codec + */ protected abstract FieldInfosFormat originalFieldInfosFormat(); // mark all fields as no term vectors, no norms, no payloads, and no vectors. @@ -170,8 +208,12 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { return segmentInfo1; } - // Special handling for Lucene8xCodecs (which are currently bundled with Lucene) - // Use BWCLucene8xCodec instead as that one extends BWCCodec (similar to all other older codecs) + /** + * Returns a backward-compatible codec for the given codec. If the codec is one of the known Lucene 8.x codecs, + * it returns a corresponding read-only backward-compatible codec. Otherwise, it returns the original codec. + * This switch if for indices created in ES 6.x, that may have some of their segments written with ES 7.x, hence Lucene 8.x. + */ + @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) private static Codec getBackwardCompatibleCodec(Codec codec) { if (codec == null) return null; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index f4e484b80d068..4c0d2c96a53d9 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -46,10 +46,9 @@ public class MetadataOnlyBKDReader extends PointValues { final long pointCount; final int docCount; final int version; + final long minLeafBlockFP; final int numIndexBytes; - final long minLeafBlockFP; - private final long indexStartPointer; public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); @@ -93,7 +92,8 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { // This code has been introduced to process IndexInput created with Lucene86Codec+ numIndexBytes = metaIn.readVInt(); minLeafBlockFP = metaIn.readLong(); - indexStartPointer = metaIn.readLong(); + // The following fields are not used in this class, but we need to read them to advance the pointer + long indexStartPointer = metaIn.readLong(); } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index 5abeb19aa7a07..e907e12716bd6 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene80.Lucene80Codec} * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 * (Elasticsearch [7.0.0-7.5.2]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ @@ -48,11 +49,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { // Needed for SPI loading @SuppressWarnings("unused") public BWCLucene80Codec() { - this("BWCLucene80Codec"); - } - - public BWCLucene80Codec(String name) { - super(name); + super("BWCLucene80Codec"); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index 1957221256041..a9d095dff9133 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene84.Lucene84Codec} * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 * (Elasticsearch [7.6.0-7.8.1]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ @@ -58,11 +59,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { // Needed for SPI loading @SuppressWarnings("unused") public BWCLucene84Codec() { - this("BWCLucene84Codec"); - } - - public BWCLucene84Codec(String name) { - super(name); + super("BWCLucene84Codec"); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); this.defaultFormat = new Lucene84PostingsFormat(); this.defaultDVFormat = new Lucene80DocValuesFormat(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 4a02f11683798..9bf2e08b958f2 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -12,7 +12,6 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; @@ -23,10 +22,10 @@ import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86Codec} * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2 * (Elasticsearch [7.9.0-7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ @@ -35,16 +34,8 @@ public class BWCLucene86Codec extends BWCCodec { private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); - private final PostingsFormat defaultFormat; private final DocValuesFormat defaultDVFormat; - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - }; - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { @@ -57,14 +48,8 @@ public DocValuesFormat getDocValuesFormatForField(String field) { // Needed for SPI loading @SuppressWarnings("unused") public BWCLucene86Codec() { - this("BWCLucene86Codec"); - } - - /** Instantiates a new codec. */ - public BWCLucene86Codec(String name) { - super(name); + super("BWCLucene86Codec"); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - this.defaultFormat = new Lucene84PostingsFormat(); this.defaultDVFormat = new Lucene80DocValuesFormat(); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index 57bb515a3a6e8..ad8a7b02b1882 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -27,32 +27,13 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.Lucene86MetadataOnlyPointsFormat; -import java.util.Objects; - /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene87.Lucene87Codec} * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.3 * (Elasticsearch [7.10.0-7-17.26]), mounted as archive indices in Elasticsearch 8.x / 9.x. */ public class BWCLucene87Codec extends BWCCodec { - private enum Mode { - /** Trade compression ratio for retrieval speed. */ - BEST_SPEED(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, Lucene80DocValuesFormat.Mode.BEST_SPEED), - /** Trade retrieval speed for compression ratio. */ - BEST_COMPRESSION(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); - - /** compression mode for stored fields */ - private final Lucene87StoredFieldsFormat.Mode storedMode; - - /** compression mode for doc value fields */ - private final Lucene80DocValuesFormat.Mode dvMode; - - Mode(Lucene87StoredFieldsFormat.Mode storedMode, Lucene80DocValuesFormat.Mode dvMode) { - this.storedMode = Objects.requireNonNull(storedMode); - this.dvMode = Objects.requireNonNull(dvMode); - } - } - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); @@ -78,14 +59,10 @@ public DocValuesFormat getDocValuesFormatForField(String field) { // Needed for SPI loading @SuppressWarnings("unused") public BWCLucene87Codec() { - this("BWCLucene87Codec", Mode.BEST_COMPRESSION); - } - - public BWCLucene87Codec(String name, Mode mode) { - super(name); - this.storedFieldsFormat = new Lucene87StoredFieldsFormat(mode.storedMode); + super("BWCLucene87Codec"); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); this.defaultFormat = new Lucene84PostingsFormat(); - this.defaultDVFormat = new Lucene80DocValuesFormat(mode.dvMode); + this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java index ac50db344c38b..219cfa29f13ce 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java @@ -18,6 +18,9 @@ import org.apache.lucene.codecs.StoredFieldsFormat; import org.elasticsearch.test.ESTestCase; +/** + * Unit tests for the {@link BWCCodec} class. + */ public class BWCCodecTests extends ESTestCase { private final Codec codec; @@ -66,14 +69,23 @@ public PointsFormat pointsFormat() { }; } + /** + * Tests that the {@link Codec#normsFormat()} method throws an {@link UnsupportedOperationException}. + */ public void testNormsFormatUnsupportedOperation() { assertThrows(UnsupportedOperationException.class, codec::normsFormat); } + /** + * Tests that the {@link Codec#termVectorsFormat()} method throws an {@link UnsupportedOperationException}. + */ public void testTermVectorsFormatUnsupportedOperation() { assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); } + /** + * Tests that the {@link Codec#knnVectorsFormat()} method throws an {@link UnsupportedOperationException}. + */ public void testKnnVectorsFormatUnsupportedOperation() { assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); } diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index 81a772904535e..11bca0624eabc 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -28,7 +28,7 @@ public class OldCodecsAvailableTests extends ESTestCase { */ @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) public void testLuceneBWCCodecsAvailable() { - assertEquals("Add Lucene BWC codecs for Elasticsearch version 9", 9, Version.CURRENT.major); + assertEquals("Add Lucene BWC codecs for Elasticsearch version 8", 9, Version.CURRENT.major); String codecPathRegex = ".*[\\\\.](Lucene(8[0-9])Codec)"; Pattern codecPathPattern = Pattern.compile(codecPathRegex); From 668ccb30e09b0b19162adf95ae9a800f0ab50004 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Tue, 11 Mar 2025 16:44:56 +0200 Subject: [PATCH 35/41] update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 23 ++++++++++++------- .../Lucene60MetadataOnlyPointsReader.java | 2 +- .../lucene60/MetadataOnlyBKDReader.java | 18 ++++++++------- .../lucene60/MetadataOnlyCodecVersion.java | 13 +++++++++++ .../bwc/codecs/lucene70/BWCLucene70Codec.java | 6 ----- .../bwc/codecs/lucene80/BWCLucene80Codec.java | 6 ----- .../bwc/codecs/lucene84/BWCLucene84Codec.java | 17 -------------- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 6 ----- .../Lucene86MetadataOnlyPointsReader.java | 3 ++- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 17 -------------- 10 files changed, 41 insertions(+), 70 deletions(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 250c963520b81..810fcc1691c5e 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -46,6 +46,7 @@ public abstract class BWCCodec extends Codec { private final FieldInfosFormat fieldInfosFormat; private final SegmentInfoFormat segmentInfosFormat; + private final PostingsFormat postingsFormat; protected BWCCodec(String name) { super(name); @@ -79,14 +80,14 @@ public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws I wrappedFormat.write(dir, info, ioContext); } }; - } - protected final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - throw new UnsupportedOperationException("Old codecs can't be used for writing"); - } - }; + this.postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + throw new UnsupportedOperationException("Old codecs can't be used for writing"); + } + }; + } @Override public final FieldInfosFormat fieldInfosFormat() { @@ -98,6 +99,11 @@ public SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + /** * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. * This method is never called in practice, as we rewrite field infos to override the info about which features are present in @@ -211,7 +217,8 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { /** * Returns a backward-compatible codec for the given codec. If the codec is one of the known Lucene 8.x codecs, * it returns a corresponding read-only backward-compatible codec. Otherwise, it returns the original codec. - * This switch if for indices created in ES 6.x, that may have some of their segments written with ES 7.x, hence Lucene 8.x. + * This switch is only for indices created in ES 6.x, later written into in ES 7.x (Lucene 8.x). Indices created + * in ES 7.x can be read directly by ES if marked read-only, without going through archive indices. */ @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) private static Codec getBackwardCompatibleCodec(Codec codec) { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java index 2e796a04200fe..c2fd3b5895907 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java @@ -105,7 +105,7 @@ public Lucene60MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc int fieldNumber = ent.getKey(); long fp = ent.getValue(); dataIn.seek(fp); - PointValues reader = new MetadataOnlyBKDReader(dataIn); + PointValues reader = new MetadataOnlyBKDReader(dataIn, MetadataOnlyCodecVersion.V_6_0); readers.put(fieldNumber, reader); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 4c0d2c96a53d9..3e7215ba38d42 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -46,11 +46,11 @@ public class MetadataOnlyBKDReader extends PointValues { final long pointCount; final int docCount; final int version; - final long minLeafBlockFP; + long minLeafBlockFP; - final int numIndexBytes; + int numIndexBytes; - public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { + public MetadataOnlyBKDReader(IndexInput metaIn, MetadataOnlyCodecVersion codecVersion) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); final int numDims = metaIn.readVInt(); final int numIndexDims; @@ -89,11 +89,13 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); - // This code has been introduced to process IndexInput created with Lucene86Codec+ - numIndexBytes = metaIn.readVInt(); - minLeafBlockFP = metaIn.readLong(); - // The following fields are not used in this class, but we need to read them to advance the pointer - long indexStartPointer = metaIn.readLong(); + if (codecVersion == MetadataOnlyCodecVersion.V_8_6) { + // This code has been introduced to process IndexInput created with Lucene86Codec+ + numIndexBytes = metaIn.readVInt(); + minLeafBlockFP = metaIn.readLong(); + // The following fields are not used in this class, but we need to read them to advance the pointer + long indexStartPointer = metaIn.readLong(); + } } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java new file mode 100644 index 0000000000000..eedf547d69dac --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene60; + +public enum MetadataOnlyCodecVersion { + V_6_0, + V_8_6 +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index f9b70d5a0e0ef..a06537dd0f820 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -17,7 +17,6 @@ import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; @@ -85,11 +84,6 @@ public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public PointsFormat pointsFormat() { return pointsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java index e907e12716bd6..9537b4e6f7fa0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -18,7 +18,6 @@ import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; @@ -68,11 +67,6 @@ public final StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; } - @Override - public final PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java index a9d095dff9133..6771f4b3130c1 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -13,17 +13,14 @@ import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; @@ -36,16 +33,8 @@ public class BWCLucene84Codec extends BWCCodec { private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final PostingsFormat defaultFormat; private final DocValuesFormat defaultDVFormat; - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - }; - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { @@ -61,7 +50,6 @@ public DocValuesFormat getDocValuesFormatForField(String field) { public BWCLucene84Codec() { super("BWCLucene84Codec"); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - this.defaultFormat = new Lucene84PostingsFormat(); this.defaultDVFormat = new Lucene80DocValuesFormat(); } @@ -80,11 +68,6 @@ public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 9bf2e08b958f2..1949285118aed 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -18,7 +18,6 @@ import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; @@ -68,11 +67,6 @@ public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 232aa6ba949d9..d46eea49f1f70 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -30,6 +30,7 @@ import org.apache.lucene.store.ChecksumIndexInput; import org.elasticsearch.core.IOUtils; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyCodecVersion; import java.io.IOException; import java.util.HashMap; @@ -73,7 +74,7 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc } else if (fieldNumber < 0) { throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn); } - PointValues reader = new MetadataOnlyBKDReader(metaIn); + PointValues reader = new MetadataOnlyBKDReader(metaIn, MetadataOnlyCodecVersion.V_8_6); readers.put(fieldNumber, reader); } indexLength = metaIn.readLong(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java index ad8a7b02b1882..f461bdee8864d 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -11,7 +11,6 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat; import org.apache.lucene.codecs.CompoundFormat; @@ -19,11 +18,9 @@ import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.Lucene86MetadataOnlyPointsFormat; @@ -37,16 +34,8 @@ public class BWCLucene87Codec extends BWCCodec { private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); - private final PostingsFormat defaultFormat; private final DocValuesFormat defaultDVFormat; - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return defaultFormat; - } - }; - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { @@ -61,7 +50,6 @@ public DocValuesFormat getDocValuesFormatForField(String field) { public BWCLucene87Codec() { super("BWCLucene87Codec"); this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); - this.defaultFormat = new Lucene84PostingsFormat(); this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); } @@ -80,11 +68,6 @@ public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public final LiveDocsFormat liveDocsFormat() { return liveDocsFormat; From 952ec0159c6f69168fdbc9c82def83ac01c2b203 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Tue, 11 Mar 2025 18:38:04 +0200 Subject: [PATCH 36/41] update after review --- .../AbstractUpgradeCompatibilityTestCase.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java index 91408540ae512..e6c1c2571c06f 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java @@ -40,6 +40,7 @@ import java.nio.file.Paths; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Consumer; import java.util.stream.Stream; @@ -218,9 +219,26 @@ private void restoreMountAndAssertSnapshot( assertTrue(getIndices(client()).contains(index)); assertDocCount(client(), index, numDocs); + assertPhraseQuery(client(), index, "Elasticsearch Doc"); assertTrue(getIndices(client()).contains(indexMount)); - assertTrue(getIndices(client()).contains(indexMount)); + } + + private static void assertPhraseQuery(RestClient client, String indexName, String phrase) throws IOException { + var request = new Request("GET", "/" + indexName + "/_search"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + request.setJsonEntity(Strings.format(""" + { + "query": { + "match": { + "content": "%s" + } + } + }""", phrase)); + Response response = client.performRequest(request); + Map map = responseAsMap(response); + int hits = ((List) ((Map) map.get("hits")).get("hits")).size(); + assertEquals("expected 1 documents but it was a different number", 1, hits); } private static String getIndices(RestClient client) throws IOException { From e911ed225589496b36564f33e26c620888f08ca3 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 14 Mar 2025 11:38:47 +0200 Subject: [PATCH 37/41] update after review --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 5 ++++- .../lucene60/Lucene60MetadataOnlyPointsFormat.java | 1 + .../lucene60/Lucene60MetadataOnlyPointsReader.java | 7 +++++-- .../bwc/codecs/lucene60/MetadataOnlyBKDReader.java | 13 +++++-------- .../codecs/lucene60/MetadataOnlyCodecVersion.java | 13 ------------- .../lucene86/Lucene86MetadataOnlyPointsFormat.java | 1 + .../lucene86/Lucene86MetadataOnlyPointsReader.java | 13 +++++++------ 7 files changed, 23 insertions(+), 30 deletions(-) delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 810fcc1691c5e..8e0f87eb3d00b 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -95,7 +95,7 @@ public final FieldInfosFormat fieldInfosFormat() { } @Override - public SegmentInfoFormat segmentInfoFormat() { + public final SegmentInfoFormat segmentInfoFormat() { return segmentInfosFormat; } @@ -217,6 +217,9 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { /** * Returns a backward-compatible codec for the given codec. If the codec is one of the known Lucene 8.x codecs, * it returns a corresponding read-only backward-compatible codec. Otherwise, it returns the original codec. + * Lucene 8.x codecs are still shipped with the current version of Lucene. + * Earlier codecs we are providing directly they will also be read-only backward-compatible, but they don't require the renaming. + * * This switch is only for indices created in ES 6.x, later written into in ES 7.x (Lucene 8.x). Indices created * in ES 7.x can be read directly by ES if marked read-only, without going through archive indices. */ diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java index fc90a3e14b944..6499f8af72bb2 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java @@ -28,6 +28,7 @@ import java.io.IOException; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat} * Allows reading metadata only from Lucene 6.0 point format **/ public class Lucene60MetadataOnlyPointsFormat extends PointsFormat { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java index c2fd3b5895907..8a5ca4acd16cb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java @@ -34,7 +34,10 @@ import java.util.HashMap; import java.util.Map; -/** Reads the metadata of point values previously written with Lucene60PointsWriter */ +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsReader} + * Reads the metadata of point values previously written with Lucene60PointsWriter + */ public final class Lucene60MetadataOnlyPointsReader extends PointsReader { final IndexInput dataIn; final SegmentReadState readState; @@ -105,7 +108,7 @@ public Lucene60MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc int fieldNumber = ent.getKey(); long fp = ent.getValue(); dataIn.seek(fp); - PointValues reader = new MetadataOnlyBKDReader(dataIn, MetadataOnlyCodecVersion.V_6_0); + PointValues reader = new MetadataOnlyBKDReader(dataIn, false); readers.put(fieldNumber, reader); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 3e7215ba38d42..38b0ed6dcdb46 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -46,11 +46,8 @@ public class MetadataOnlyBKDReader extends PointValues { final long pointCount; final int docCount; final int version; - long minLeafBlockFP; - int numIndexBytes; - - public MetadataOnlyBKDReader(IndexInput metaIn, MetadataOnlyCodecVersion codecVersion) throws IOException { + public MetadataOnlyBKDReader(IndexInput metaIn, boolean isCodec86Version) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); final int numDims = metaIn.readVInt(); final int numIndexDims; @@ -89,12 +86,12 @@ public MetadataOnlyBKDReader(IndexInput metaIn, MetadataOnlyCodecVersion codecVe pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); - if (codecVersion == MetadataOnlyCodecVersion.V_8_6) { + if (isCodec86Version) { // This code has been introduced to process IndexInput created with Lucene86Codec+ - numIndexBytes = metaIn.readVInt(); - minLeafBlockFP = metaIn.readLong(); + metaIn.readVInt(); + metaIn.readLong(); // The following fields are not used in this class, but we need to read them to advance the pointer - long indexStartPointer = metaIn.readLong(); + metaIn.readLong(); } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java deleted file mode 100644 index eedf547d69dac..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyCodecVersion.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene60; - -public enum MetadataOnlyCodecVersion { - V_6_0, - V_8_6 -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java index 881061273f7e9..f7902c5c9e2a0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java @@ -28,6 +28,7 @@ import java.io.IOException; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat} * Allows reading metadata only from Lucene 8.6 point format **/ public class Lucene86MetadataOnlyPointsFormat extends PointsFormat { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index d46eea49f1f70..4f1fb4b1abddb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -30,13 +30,15 @@ import org.apache.lucene.store.ChecksumIndexInput; import org.elasticsearch.core.IOUtils; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; -import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyCodecVersion; import java.io.IOException; import java.util.HashMap; import java.util.Map; -/** Reads the metadata of point values previously written with Lucene86PointsWriter */ +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsReader} + * Reads the metadata of point values previously written with Lucene86PointsWriter + */ public final class Lucene86MetadataOnlyPointsReader extends PointsReader { final SegmentReadState readState; final Map readers = new HashMap<>(); @@ -52,7 +54,6 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc boolean success = false; try { - long indexLength = -1, dataLength = -1; try ( ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context) ) { @@ -74,11 +75,11 @@ public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc } else if (fieldNumber < 0) { throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn); } - PointValues reader = new MetadataOnlyBKDReader(metaIn, MetadataOnlyCodecVersion.V_8_6); + PointValues reader = new MetadataOnlyBKDReader(metaIn, true); readers.put(fieldNumber, reader); } - indexLength = metaIn.readLong(); - dataLength = metaIn.readLong(); + metaIn.readLong(); + metaIn.readLong(); } catch (Throwable t) { priorE = t; } finally { From 44cd893c7ebfcacf5a35567def5d87d1cb79c0ba Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 14 Mar 2025 13:13:18 +0200 Subject: [PATCH 38/41] minor naming update after review --- .../org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java | 4 +++- .../lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 8e0f87eb3d00b..fb98c45916440 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -29,6 +29,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Version; import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; @@ -193,7 +194,7 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { public static SegmentInfo wrap(SegmentInfo segmentInfo) { Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec()); - + final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) @@ -228,6 +229,7 @@ private static Codec getBackwardCompatibleCodec(Codec codec) { if (codec == null) return null; return switch (codec.getClass().getSimpleName()) { + case "Lucene70Codec" -> new BWCLucene70Codec(); case "Lucene80Codec" -> new BWCLucene80Codec(); case "Lucene84Codec" -> new BWCLucene84Codec(); case "Lucene86Codec" -> new BWCLucene86Codec(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 38b0ed6dcdb46..3beb4b9c4654c 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -47,7 +47,7 @@ public class MetadataOnlyBKDReader extends PointValues { final int docCount; final int version; - public MetadataOnlyBKDReader(IndexInput metaIn, boolean isCodec86Version) throws IOException { + public MetadataOnlyBKDReader(IndexInput metaIn, boolean isVersionPost86) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); final int numDims = metaIn.readVInt(); final int numIndexDims; @@ -86,8 +86,8 @@ public MetadataOnlyBKDReader(IndexInput metaIn, boolean isCodec86Version) throws pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); - if (isCodec86Version) { - // This code has been introduced to process IndexInput created with Lucene86Codec+ + // This code has been introduced to process IndexInput created with Lucene86Codec+ + if (isVersionPost86) { metaIn.readVInt(); metaIn.readLong(); // The following fields are not used in this class, but we need to read them to advance the pointer From b9eb51682436e84554f3b1a32fcb3e3f4286b41a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 14 Mar 2025 11:21:15 +0000 Subject: [PATCH 39/41] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index fb98c45916440..f12935a2bec08 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -194,7 +194,7 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { public static SegmentInfo wrap(SegmentInfo segmentInfo) { Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec()); - + final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) From 36f62ca36de71fc84bb8f79383717553685f3690 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 20 Mar 2025 12:43:21 +0200 Subject: [PATCH 40/41] update after review --- .../lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java | 3 ++- .../bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 3beb4b9c4654c..ab865cba8722e 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -86,7 +86,8 @@ public MetadataOnlyBKDReader(IndexInput metaIn, boolean isVersionPost86) throws pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); - // This code has been introduced to process IndexInput created with Lucene86Codec+ + // This code has been introduced to process IndexInput created with Lucene86Codec+. This is not necessary + // in the read-only version for older formats. if (isVersionPost86) { metaIn.readVInt(); metaIn.readLong(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 4f1fb4b1abddb..8e9d7196f29e2 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -108,9 +108,10 @@ public PointValues getValues(String fieldName) { return readers.get(fieldInfo.number); } + // We only open the metadata field, and do nothing with the other two files (index/data), + // for which Lucene checks integrity but we don't need to. @Override public void checkIntegrity() { - ; } @Override From 2952e62e814ec0521efd355ad07a491f4930baa2 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 20 Mar 2025 10:53:00 +0000 Subject: [PATCH 41/41] [CI] Auto commit changes from spotless --- .../bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java index 8e9d7196f29e2..55671828b4dcd 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -111,8 +111,7 @@ public PointValues getValues(String fieldName) { // We only open the metadata field, and do nothing with the other two files (index/data), // for which Lucene checks integrity but we don't need to. @Override - public void checkIntegrity() { - } + public void checkIntegrity() {} @Override public void close() throws IOException {