Support 7x segments as archive in 8x / 9x

docs/changelog/119503.yaml

@@ -0,0 +1,6 @@
+pr: 119503
+summary: Support 7x segments as archive in 8x / 9x
+area: Search
+type: bug
+issues:
+ - 117042

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java

@@ -17,6 +17,7 @@
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
@@ -26,6 +27,11 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Version;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -41,18 +47,25 @@ protected BWCCodec(String name) {
         super(name);
     }
 
+    protected final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+        @Override
+        public PostingsFormat getPostingsFormatForField(String field) {
+            throw new UnsupportedOperationException("Old codecs can't be used for writing");
+        }
+    };
+
     @Override
-    public NormsFormat normsFormat() {
+    public final NormsFormat normsFormat() {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public TermVectorsFormat termVectorsFormat() {
+    public final TermVectorsFormat termVectorsFormat() {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public KnnVectorsFormat knnVectorsFormat() {
+    public final KnnVectorsFormat knnVectorsFormat() {
         throw new UnsupportedOperationException();
     }
 
@@ -118,13 +131,14 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) {
     }
 
     public static SegmentInfo wrap(SegmentInfo segmentInfo) {
-        final Codec codec = segmentInfo.getCodec();
+        Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec());
+
         final SegmentInfo segmentInfo1 = new SegmentInfo(
             segmentInfo.dir,
             // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation)
             // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed?
-            org.apache.lucene.util.Version.LATEST,
-            org.apache.lucene.util.Version.LATEST,
+            Version.LATEST,
+            Version.LATEST,
             segmentInfo.name,
             segmentInfo.maxDoc(),
             segmentInfo.getUseCompoundFile(),
@@ -139,6 +153,20 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) {
         return segmentInfo1;
     }
 
+    // Special handling for Lucene8xCodecs (which are currently bundled with Lucene)
+    // Use BWCLucene8xCodec instead as that one extends BWCCodec (similar to all other older codecs)
+    private static Codec getBackwardCompatibleCodec(Codec codec) {
+        if (codec == null) return null;
+
+        return switch (codec.getClass().getSimpleName()) {
+            case "Lucene80Codec" -> new BWCLucene80Codec();
+            case "Lucene84Codec" -> new BWCLucene84Codec();
+            case "Lucene86Codec" -> new BWCLucene86Codec();
+            case "Lucene87Codec" -> new BWCLucene87Codec();
+            default -> codec;
+        };
+    }
+
     /**
      * In-memory postings format that shows no postings available.
      */

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java

@@ -47,6 +47,10 @@ public class MetadataOnlyBKDReader extends PointValues {
     final int docCount;
     final int version;
 
+    final int numIndexBytes;
+    final long minLeafBlockFP;
+    private final long indexStartPointer;
+
     public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException {
         version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT);
         final int numDims = metaIn.readVInt();
@@ -85,6 +89,16 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException {
 
         pointCount = metaIn.readVLong();
         docCount = metaIn.readVInt();
+
+        numIndexBytes = metaIn.readVInt();
+        if (version >= VERSION_META_FILE) {
+            minLeafBlockFP = metaIn.readLong();
+            indexStartPointer = metaIn.readLong();
+        } else {
+            indexStartPointer = metaIn.getFilePointer();
+            minLeafBlockFP = metaIn.readVLong();
+            metaIn.seek(indexStartPointer);
+        }
     }
 
     @Override

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java

@@ -21,7 +21,6 @@
 import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
 import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat;
 
@@ -45,12 +44,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) {
             return defaultDVFormat;
         }
     };
-    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
-        @Override
-        public PostingsFormat getPostingsFormatForField(String field) {
-            throw new IllegalStateException("This codec should only be used for reading, not writing");
-        }
-    };
+    private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat();
 
     // Needed for SPI loading
     @SuppressWarnings("unused")
@@ -100,6 +94,6 @@ public PostingsFormat postingsFormat() {
 
     @Override
     public PointsFormat pointsFormat() {
-        return new Lucene60MetadataOnlyPointsFormat();
+        return pointsFormat;
     }
 }

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat;
+
+/**
+ * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0
+ * (Elasticsearch [7.0.0-7.5.2]), mounted as archive indices in Elasticsearch 8.x / 9.x.
+ */
+public class BWCLucene80Codec extends BWCCodec {
+
+    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
+    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat());
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return defaultDVFormat;
+        }
+    };
+    private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat();
+
+    private final StoredFieldsFormat storedFieldsFormat;
+    private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat();
+
+    // Needed for SPI loading
+    @SuppressWarnings("unused")
+    public BWCLucene80Codec() {
+        this("BWCLucene80Codec");
+    }
+
+    public BWCLucene80Codec(String name) {
+        super(name);
+        this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
+    }
+
+    @Override
+    public final StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    @Override
+    public final FieldInfosFormat fieldInfosFormat() {
+        return fieldInfosFormat;
+    }
+
+    @Override
+    public final SegmentInfoFormat segmentInfoFormat() {
+        return segmentInfosFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public final CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public final PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat;
+
+/**
+ * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1
+ * (Elasticsearch [7.6.0-7.8.1]), mounted as archive indices in Elasticsearch 8.x / 9.x.
+ */
+public class BWCLucene84Codec extends BWCCodec {
+
+    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
+    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat());
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+    private final PostingsFormat defaultFormat;
+
+    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+        @Override
+        public PostingsFormat getPostingsFormatForField(String field) {
+            return BWCLucene84Codec.this.getPostingsFormatForField(field);
+        }
+    };
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return BWCLucene84Codec.this.getDocValuesFormatForField(field);
+        }
+    };
+
+    private final StoredFieldsFormat storedFieldsFormat;
+    private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat();
+
+    // Needed for SPI loading
+    @SuppressWarnings("unused")
+    public BWCLucene84Codec() {
+        this("BWCLucene84Codec");
+    }
+
+    public BWCLucene84Codec(String name) {
+        super(name);
+        this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
+        this.defaultFormat = new Lucene84PostingsFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    @Override
+    public final FieldInfosFormat fieldInfosFormat() {
+        return fieldInfosFormat;
+    }
+
+    @Override
+    public SegmentInfoFormat segmentInfoFormat() {
+        return segmentInfosFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    /**
+     * Returns the postings format that should be used for writing new segments of <code>field</code>.
+     *
+     * <p>The default implementation always returns "Lucene84".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public PostingsFormat getPostingsFormatForField(String field) {
+        return defaultFormat;
+    }
+
+    /**
+     * Returns the docvalues format that should be used for writing new segments of <code>field</code>
+     * .
+     *
+     * <p>The default implementation always returns "Lucene80".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return defaultDVFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+
+    private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat();
+}