Skip to content

Commit ca91219

Browse files
javannajavanna
authored
Bump Elasticsearch codec to track Lucene101Codec (#116318)
 See apache/lucene#13968
1 parent 43d280e commit ca91219

26 files changed

+193
-55
lines changed

‎server/src/main/java/module-info.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,8 @@
460460
with
461461
org.elasticsearch.index.codec.Elasticsearch814Codec,
462462
org.elasticsearch.index.codec.Elasticsearch816Codec,
463-
org.elasticsearch.index.codec.Elasticsearch900Codec;
463+
org.elasticsearch.index.codec.Elasticsearch900Codec,
464+
org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
464465

465466
provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider;
466467

‎server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat;
1414
import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
1515
import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
16+
import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
1617
import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
1718
import org.apache.lucene.codecs.DocValuesProducer;
1819
import org.apache.lucene.codecs.FieldsProducer;
@@ -21,7 +22,7 @@
2122
import org.apache.lucene.codecs.PointsReader;
2223
import org.apache.lucene.codecs.StoredFieldsReader;
2324
import org.apache.lucene.codecs.TermVectorsReader;
24-
import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
25+
import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
2526
import org.apache.lucene.index.BinaryDocValues;
2627
import org.apache.lucene.index.ByteVectorValues;
2728
import org.apache.lucene.index.DirectoryReader;
@@ -306,6 +307,9 @@ private static void readProximity(Terms terms, PostingsEnum postings) throws IOE
306307
private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException {
307308
if (term != null && termsEnum.seekExact(term)) {
308309
final TermState termState = termsEnum.termState();
310+
if (termState instanceof final Lucene101PostingsFormat.IntBlockTermState blockTermState) {
311+
return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP);
312+
}
309313
if (termState instanceof final Lucene912PostingsFormat.IntBlockTermState blockTermState) {
310314
return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP);
311315
}

‎server/src/main/java/org/elasticsearch/common/lucene/Lucene.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@
8888
import java.util.Objects;
8989

9090
public class Lucene {
91-
public static final String LATEST_CODEC = "Lucene100";
91+
public static final String LATEST_CODEC = "Lucene101";
9292

9393
public static final String SOFT_DELETES_FIELD = "__soft_deletes";
9494

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ private static Version parseUnchecked(String version) {
132132
public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT = def(8_519_00_0, Version.LUCENE_9_12_0);
133133
public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
134134
public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
135-
public static final IndexVersion UPGRADE_TO_LUCENE_10_0_1 = def(9_002_00_0, Version.LUCENE_10_0_1);
135+
public static final IndexVersion UPGRADE_TO_LUCENE_10_1_0 = def(9_002_00_0, Version.LUCENE_10_1_0);
136136

137137
/*
138138
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/index/codec/CodecService.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import org.apache.lucene.codecs.Codec;
1313
import org.apache.lucene.codecs.FieldInfosFormat;
1414
import org.apache.lucene.codecs.FilterCodec;
15-
import org.apache.lucene.codecs.lucene100.Lucene100Codec;
15+
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
1616
import org.elasticsearch.common.util.BigArrays;
1717
import org.elasticsearch.common.util.FeatureFlag;
1818
import org.elasticsearch.core.Nullable;
@@ -46,7 +46,7 @@ public class CodecService implements CodecProvider {
4646
public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
4747
final var codecs = new HashMap<String, Codec>();
4848

49-
Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_SPEED, mapperService, bigArrays);
49+
Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, bigArrays);
5050
if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
5151
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays));
5252
} else {
@@ -58,7 +58,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)
5858
BEST_COMPRESSION_CODEC,
5959
new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
6060
);
61-
Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
61+
Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene101Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
6262
codecs.put(LEGACY_BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
6363

6464
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());

‎server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@
1010
package org.elasticsearch.index.codec;
1111

1212
import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec;
13+
import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
1314
import org.apache.lucene.codecs.DocValuesFormat;
1415
import org.apache.lucene.codecs.KnnVectorsFormat;
1516
import org.apache.lucene.codecs.PostingsFormat;
1617
import org.apache.lucene.codecs.StoredFieldsFormat;
1718
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
18-
import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
1919
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
2020
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
2121
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;

‎server/src/main/java/org/elasticsearch/index/codec/Elasticsearch900Codec.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,22 @@
99

1010
package org.elasticsearch.index.codec;
1111

12+
import org.apache.lucene.backward_codecs.lucene100.Lucene100Codec;
13+
import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
1214
import org.apache.lucene.codecs.DocValuesFormat;
1315
import org.apache.lucene.codecs.KnnVectorsFormat;
1416
import org.apache.lucene.codecs.PostingsFormat;
1517
import org.apache.lucene.codecs.StoredFieldsFormat;
16-
import org.apache.lucene.codecs.lucene100.Lucene100Codec;
1718
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
18-
import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
1919
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
2020
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
2121
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
2222
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
2323
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
2424

2525
/**
26-
* Elasticsearch codec as of 9.0. This extends the Lucene 10.0 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See
27-
* {@link Zstd814StoredFieldsFormat}.
26+
* Elasticsearch codec as of 9.0-snapshot relying on Lucene 10.0. This extends the Lucene 10.0 codec to compressed stored fields
27+
* with ZSTD instead of LZ4/DEFLATE. See {@link Zstd814StoredFieldsFormat}.
2828
*/
2929
public class Elasticsearch900Codec extends CodecService.DeduplicateFieldInfosCodec {
3030

‎server/src/main/java/org/elasticsearch/index/codec/Elasticsearch900Lucene101Codec.java‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.codec;
11+
12+
import org.apache.lucene.codecs.DocValuesFormat;
13+
import org.apache.lucene.codecs.KnnVectorsFormat;
14+
import org.apache.lucene.codecs.PostingsFormat;
15+
import org.apache.lucene.codecs.StoredFieldsFormat;
16+
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
17+
import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
18+
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
19+
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
20+
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
21+
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
22+
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
23+
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
24+
25+
/**
26+
* Elasticsearch codec as of 9.0 relying on Lucene 10.1. This extends the Lucene 10.1 codec to compressed
27+
* stored fields with ZSTD instead of LZ4/DEFLATE. See {@link Zstd814StoredFieldsFormat}.
28+
*/
29+
public class Elasticsearch900Lucene101Codec extends CodecService.DeduplicateFieldInfosCodec {
30+
31+
private final StoredFieldsFormat storedFieldsFormat;
32+
33+
private final PostingsFormat defaultPostingsFormat;
34+
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
35+
@Override
36+
public PostingsFormat getPostingsFormatForField(String field) {
37+
return Elasticsearch900Lucene101Codec.this.getPostingsFormatForField(field);
38+
}
39+
};
40+
41+
private final DocValuesFormat defaultDVFormat;
42+
private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
43+
@Override
44+
public DocValuesFormat getDocValuesFormatForField(String field) {
45+
return Elasticsearch900Lucene101Codec.this.getDocValuesFormatForField(field);
46+
}
47+
};
48+
49+
private final KnnVectorsFormat defaultKnnVectorsFormat;
50+
private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() {
51+
@Override
52+
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
53+
return Elasticsearch900Lucene101Codec.this.getKnnVectorsFormatForField(field);
54+
}
55+
};
56+
57+
/** Public no-arg constructor, needed for SPI loading at read-time. */
58+
public Elasticsearch900Lucene101Codec() {
59+
this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
60+
}
61+
62+
/**
63+
* Constructor. Takes a {@link Zstd814StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of
64+
* worse space-efficiency or vice-versa.
65+
*/
66+
public Elasticsearch900Lucene101Codec(Zstd814StoredFieldsFormat.Mode mode) {
67+
super("Elasticsearch900Lucene101", new Lucene101Codec());
68+
this.storedFieldsFormat = mode.getFormat();
69+
this.defaultPostingsFormat = new Lucene101PostingsFormat();
70+
this.defaultDVFormat = new Lucene90DocValuesFormat();
71+
this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
72+
}
73+
74+
@Override
75+
public StoredFieldsFormat storedFieldsFormat() {
76+
return storedFieldsFormat;
77+
}
78+
79+
@Override
80+
public final PostingsFormat postingsFormat() {
81+
return postingsFormat;
82+
}
83+
84+
@Override
85+
public final DocValuesFormat docValuesFormat() {
86+
return docValuesFormat;
87+
}
88+
89+
@Override
90+
public final KnnVectorsFormat knnVectorsFormat() {
91+
return knnVectorsFormat;
92+
}
93+
94+
/**
95+
* Returns the postings format that should be used for writing new segments of <code>field</code>.
96+
*
97+
* <p>The default implementation always returns "Lucene912".
98+
*
99+
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
100+
* future version of Lucene are only guaranteed to be able to read the default implementation,
101+
*/
102+
public PostingsFormat getPostingsFormatForField(String field) {
103+
return defaultPostingsFormat;
104+
}
105+
106+
/**
107+
* Returns the docvalues format that should be used for writing new segments of <code>field</code>
108+
* .
109+
*
110+
* <p>The default implementation always returns "Lucene912".
111+
*
112+
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
113+
* future version of Lucene are only guaranteed to be able to read the default implementation.
114+
*/
115+
public DocValuesFormat getDocValuesFormatForField(String field) {
116+
return defaultDVFormat;
117+
}
118+
119+
/**
120+
* Returns the vectors format that should be used for writing new segments of <code>field</code>
121+
*
122+
* <p>The default implementation always returns "Lucene912".
123+
*
124+
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
125+
* future version of Lucene are only guaranteed to be able to read the default implementation.
126+
*/
127+
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
128+
return defaultKnnVectorsFormat;
129+
}
130+
131+
}

‎server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import org.apache.lucene.codecs.DocValuesFormat;
1414
import org.apache.lucene.codecs.KnnVectorsFormat;
1515
import org.apache.lucene.codecs.PostingsFormat;
16-
import org.apache.lucene.codecs.lucene100.Lucene100Codec;
16+
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
1717
import org.elasticsearch.common.lucene.Lucene;
1818
import org.elasticsearch.common.util.BigArrays;
1919
import org.elasticsearch.index.mapper.MapperService;
@@ -22,11 +22,11 @@
2222
* Legacy version of {@link PerFieldMapperCodec}. This codec is preserved to give an escape hatch in case we encounter issues with new
2323
* changes in {@link PerFieldMapperCodec}.
2424
*/
25-
public final class LegacyPerFieldMapperCodec extends Lucene100Codec {
25+
public final class LegacyPerFieldMapperCodec extends Lucene101Codec {
2626

2727
private final PerFieldFormatSupplier formatSupplier;
2828

29-
public LegacyPerFieldMapperCodec(Lucene100Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
29+
public LegacyPerFieldMapperCodec(Lucene101Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
3030
super(compressionMode);
3131
this.formatSupplier = new PerFieldFormatSupplier(mapperService, bigArrays);
3232
// If the below assertion fails, it is a sign that Lucene released a new codec. You must create a copy of the current Elasticsearch

‎server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
* per index in real time via the mapping API. If no specific postings format or vector format is
2727
* configured for a specific field the default postings or vector format is used.
2828
*/
29-
public final class PerFieldMapperCodec extends Elasticsearch900Codec {
29+
public final class PerFieldMapperCodec extends Elasticsearch900Lucene101Codec {
3030

3131
private final PerFieldFormatSupplier formatSupplier;
3232

0 commit comments

Comments
 (0)