Docs and simplifications to support for Lucene ancient versions (elastic#124053)

The old lucene versions plugin allows users to read indices created by ancient
Elasticsearch version, starting from 5.0. Especially for 5.x which relied on Lucene
6.x, some special logic is required around postings format support. That revolves
around reading of FSTs, but has a consequence of requiring quite a few fork of
other Lucene classes due to their visibility.

This commit attempts to add javadocs to clarify the intent of some of these classes.
It also includes some simplifications, in that Lucene50PostingsReader can be reused as-is
and some other classes are only needed in tests hence are moved to the test folder.

Author: javanna

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/LegacyAdaptingPerFieldPostingsFormat.java

@@ -21,26 +21,22 @@
 
 import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene50.BWCLucene50PostingsFormat;
 
-import java.io.Closeable;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -52,12 +48,11 @@
  * latter only supports Lucene 7 and above (as it was shipped with backwards-codecs of Lucene 9 that
  * only has support for N-2).
  *
- * This class can probably be removed once we are on Lucene 10 and Lucene50PostingsFormat is no longer
+ * This class can be removed once Elasticsearch gets upgraded to Lucene 11 and Lucene50PostingsFormat is no longer
  * shipped as part of bwc jars.
- *
- * Swapping out formats can be done via the {@link #getPostingsFormat(String) method}.
  */
-public abstract class LegacyAdaptingPerFieldPostingsFormat extends PostingsFormat {
+@UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)
+public final class LegacyAdaptingPerFieldPostingsFormat extends PostingsFormat {
     /** Name of this {@link PostingsFormat}. */
     public static final String PER_FIELD_NAME = "PerField40";
 
@@ -68,39 +63,19 @@ public abstract class LegacyAdaptingPerFieldPostingsFormat extends PostingsForma
     public static final String PER_FIELD_SUFFIX_KEY = PerFieldPostingsFormat.class.getSimpleName() + ".suffix";
 
     /** Sole constructor. */
-    protected LegacyAdaptingPerFieldPostingsFormat() {
+    public LegacyAdaptingPerFieldPostingsFormat() {
         super(PER_FIELD_NAME);
     }
 
     static String getSuffix(String formatName, String suffix) {
         return formatName + "_" + suffix;
     }
 
-    protected PostingsFormat getPostingsFormat(String formatName) {
-        throw new IllegalArgumentException(formatName);
-    }
-
-    private class FieldsWriter extends FieldsConsumer {
-        final SegmentWriteState writeState;
-        final List<Closeable> toClose = new ArrayList<Closeable>();
-
-        FieldsWriter(SegmentWriteState writeState) {
-            this.writeState = writeState;
-        }
-
-        @Override
-        public void write(Fields fields, NormsProducer norms) throws IOException {
-            throw new IllegalStateException("This codec should only be used for reading, not writing");
-        }
-
-        @Override
-        public void merge(MergeState mergeState, NormsProducer norms) throws IOException {
-            throw new IllegalStateException("This codec should only be used for reading, not writing");
-        }
-
-        @Override
-        public void close() throws IOException {
-            IOUtils.close(toClose);
+    private static PostingsFormat getPostingsFormat(String formatName) {
+        if (formatName.equals("Lucene50")) {
+            return new BWCLucene50PostingsFormat();
+        } else {
+            return new BWCCodec.EmptyPostingsFormat();
         }
     }
 
@@ -130,8 +105,7 @@ private static class FieldsReader extends FieldsProducer {
             segment = other.segment;
         }
 
-        FieldsReader(final SegmentReadState readState, LegacyAdaptingPerFieldPostingsFormat legacyAdaptingPerFieldPostingsFormat)
-            throws IOException {
+        FieldsReader(final SegmentReadState readState) throws IOException {
 
             // Read _X.per and init each format:
             boolean success = false;
@@ -147,7 +121,7 @@ private static class FieldsReader extends FieldsProducer {
                             if (suffix == null) {
                                 throw new IllegalStateException("missing attribute: " + PER_FIELD_SUFFIX_KEY + " for field: " + fieldName);
                             }
-                            PostingsFormat format = legacyAdaptingPerFieldPostingsFormat.getPostingsFormat(formatName);
+                            PostingsFormat format = getPostingsFormat(formatName);
                             String segmentSuffix = getSuffix(formatName, suffix);
                             if (formats.containsKey(segmentSuffix) == false) {
                                 formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
@@ -206,12 +180,12 @@ public String toString() {
     }
 
     @Override
-    public final FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        return new FieldsWriter(state);
+    public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
+        throw new IllegalStateException("This codec should only be used for reading, not writing");
     }
 
     @Override
-    public final FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
-        return new FieldsReader(state, this);
+    public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+        return new FieldsReader(state);
     }
 }

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/CompressionAlgorithm.java

@@ -25,7 +25,10 @@
 
 import java.io.IOException;
 
-/** Compression algorithm used for suffixes of a block of terms. */
+/**
+ * This is a copy of the class with same name shipped with Lucene, which is though package protected hence not accessible.
+ * We need to copy it because we have our own fork of {@link FieldReader}.
+ */
 enum CompressionAlgorithm {
     NO_COMPRESSION(0x00) {
 

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/FieldReader.java

@@ -35,6 +35,9 @@
 
 /**
  * BlockTree's implementation of {@link Terms}.
+ *
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene40.blocktree.FieldReader} that allows to read from ancient
+ * Lucene versions. The key difference is the FST loading which relies on {@link FST}.
  */
 public final class FieldReader extends Terms {
 

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/IntersectTermsEnum.java

@@ -23,7 +23,6 @@
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -39,11 +38,8 @@
 import java.io.IOException;
 
 /**
- * This is used to implement efficient {@link Terms#intersect} for block-tree. Note that it cannot
- * seek, except for the initial term on init. It just "nexts" through the intersection of the
- * automaton and the terms. It does not use the terms index at all: on init, it loads the root
- * block, and scans its way to the initial term. Likewise, in next it scans until it finds a term
- * that matches the current automaton transition.
+ * This is a copy of the class with same name shipped with Lucene, which is though package protected hence not accessible.
+ * We need to copy it because we have our own fork of {@link FieldReader}.
  */
 final class IntersectTermsEnum extends BaseTermsEnum {
 

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/IntersectTermsEnumFrame.java

@@ -31,7 +31,10 @@
 import java.io.IOException;
 import java.util.Arrays;
 
-// TODO: can we share this with the frame in STE?
+/**
+ * This is a copy of the class with same name shipped with Lucene, which is though package protected hence not accessible.
+ * We need to copy it because we have our own fork of {@link FieldReader}.
+ */
 final class IntersectTermsEnumFrame {
     final int ord;
     long fp;

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.java

@@ -252,6 +252,10 @@ public Lucene40BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentRe
                         final long indexStartFP = indexMetaIn.readVLong();
                         FieldReader previous = fieldMap.put(
                             fieldInfo.name,
+                            /*
+                            The FieldReader used differs from the original Lucene variant, in that it is more flexible
+                            around the versions it can read from.
+                             */
                             new FieldReader(
                                 this,
                                 fieldInfo,

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/SegmentTermsEnum.java

@@ -36,7 +36,10 @@
 import java.io.IOException;
 import java.io.PrintStream;
 
-/** Iterates through terms in this field. */
+/**
+ * This is a copy of the class with same name shipped with Lucene, which is though package protected hence not accessible.
+ * We need to copy it because we have our own fork of {@link FieldReader}.
+ */
 final class SegmentTermsEnum extends BaseTermsEnum {
 
     // Lazy init:

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/SegmentTermsEnumFrame.java

@@ -31,6 +31,10 @@
 import java.io.IOException;
 import java.util.Arrays;
 
+/**
+ * This is a copy of the class with same name shipped with Lucene, which is though package protected hence not accessible.
+ * We need to copy it because we have our own fork of {@link FieldReader}.
+ */
 final class SegmentTermsEnumFrame {
     // Our index in stack[]:
     final int ord;

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene40/blocktree/Stats.java

@@ -30,7 +30,8 @@
 import java.util.Locale;
 
 /**
- * BlockTree statistics for a single field returned by {@link FieldReader#getStats()}.
+ * This is a copy of {@link Stats} shipped with Lucene, which has though package protected constructor and methods.
+ * We need to copy it because we have our own fork of {@link FieldReader}.
  */
 public class Stats {
     /** Byte size of the index. */