Skip to content
Open
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add pointer based lag metric in pull-based ingestion ([#19635](https://github.com/opensearch-project/OpenSearch/pull/19635))

### Changed
- Combining filter rewrite and skip list to optimize sub aggregation([#19573](https://github.com/opensearch-project/OpenSearch/pull/19573))
- Faster `terms` query creation for `keyword` field with index and docValues enabled ([#19350](https://github.com/opensearch-project/OpenSearch/pull/19350))
- Refactor to move prepareIndex and prepareDelete methods to Engine class ([#19551](https://github.com/opensearch-project/OpenSearch/pull/19551))
- Omit maxScoreCollector in SimpleTopDocsCollectorContext when concurrent segment search enabled ([#19584](https://github.com/opensearch-project/OpenSearch/pull/19584))
Expand Down
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[versions]
opensearch = "3.4.0"
lucene = "10.3.1"
lucene = "10.4.0-snapshot-a8c1455"

bundled_jdk_vendor = "adoptium"
bundled_jdk = "24.0.2+12"
Expand Down
1 change: 0 additions & 1 deletion libs/core/licenses/lucene-core-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
96781fdefe4a7e2ac3482ccb41801b2cd1607501
2 changes: 1 addition & 1 deletion libs/core/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_3_2_0 = new Version(3020099, org.apache.lucene.util.Version.LUCENE_10_2_2);
public static final Version V_3_3_0 = new Version(3030099, org.apache.lucene.util.Version.LUCENE_10_3_1);
public static final Version V_3_3_1 = new Version(3030199, org.apache.lucene.util.Version.LUCENE_10_3_1);
public static final Version V_3_4_0 = new Version(3040099, org.apache.lucene.util.Version.LUCENE_10_3_1);
public static final Version V_3_4_0 = new Version(3040099, org.apache.lucene.util.Version.LUCENE_10_4_0);
public static final Version CURRENT = V_3_4_0;

public static Version fromId(int id) {
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
886e976447a505c02e636ba67b4d381b257a3f44

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2ad93f9924ecc4491b26dc540bef1fabe2afd2ea

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6efc194346481fb1ca2a174182ad2429294021bb

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
11a5ecc992d8725bfb760925f07fe13a40ce4b06

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7821a0b53f5c3e25418659163968f60327484da1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6bff6a2e336dee93a4dff882138296d80ab62fd6

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5d76ae18c4e5119845cd278ed276929505c88bfe

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9513d1dcd7dd057a9026fa3c0764d778540d3b8d
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7142df4d1076f734aabd863fc56516b58dcae605
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3d9f4cf13f20e1b384021a808d8f57a73427287e
1 change: 0 additions & 1 deletion server/licenses/lucene-core-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
96781fdefe4a7e2ac3482ccb41801b2cd1607501
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
beba73c7a31f7f72d21a1485b200d864706638b3
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
077fca639cabd72a1c8521e6e3e8aacc36d0b7eb
1 change: 0 additions & 1 deletion server/licenses/lucene-join-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
79ab48828901f08ca79b74fd0d1de855a7ed8cda
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
de41ae03a22b20fa52140f178ae62cad7724cf4e
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7ae8474b4a17d92394126f9c6cc8e76e479df388
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
24e09399d8c70f5a6a0d72546cec522696bf4bac
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
86cf4d847c39a0081d23535be84bf36dbb6a262a
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c1078394a5db5510b3b8d508440d53be642fc1eb
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8b7925295d45c23b9fc087e9762afc74da578ae5
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
055860e2000dfc62982ccb1e2ba4ea8bf75acd8e
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-10.3.1.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
197e5f4d7f5dab0442c1865fce9dd941d7620022
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@
* @opensearch.internal
*/
public class Lucene {
public static final String LATEST_CODEC = "Lucene103";
public static final String LATEST_CODEC = "Lucene104";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";
public static final String PARENT_FIELD = "__nested_parent";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.opensearch.common.Nullable;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.collect.MapBuilder;
Expand Down Expand Up @@ -71,23 +71,23 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
assert null != indexSettings;
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene103Codec());
codecs.put(LZ4, new Lucene103Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene104Codec());
codecs.put(LZ4, new Lucene104Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene104Codec(Lucene104Codec.Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene104Codec(Lucene104Codec.Mode.BEST_COMPRESSION));
} else {
// CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
// We can still support all the compression codecs when composite index is present
if (mapperService.isCompositeIndexPresent()) {
codecs.putAll(compositeCodecFactory.getCompositeIndexCodecs(mapperService, logger));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(
BEST_COMPRESSION_CODEC,
new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger)
new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger)
);
codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger));
}
}
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat;
Expand All @@ -59,7 +59,7 @@
*
* @opensearch.internal
*/
public class PerFieldMappingPostingFormatCodec extends Lucene103Codec {
public class PerFieldMappingPostingFormatCodec extends Lucene104Codec {
private final Logger logger;
private final MapperService mapperService;
private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.composite.backward_codecs.composite101.Composite101Codec;
import org.opensearch.index.codec.composite.composite103.Composite103Codec;
import org.opensearch.index.codec.composite.composite104.Composite104Codec;
import org.opensearch.index.mapper.MapperService;

import java.util.HashMap;
Expand All @@ -33,16 +32,16 @@
public class CompositeCodecFactory {

// we can use this to track the latest composite codec
public static final String COMPOSITE_CODEC = Composite101Codec.COMPOSITE_INDEX_CODEC_NAME;
public static final String COMPOSITE_CODEC = Composite104Codec.COMPOSITE_INDEX_CODEC_NAME;

public CompositeCodecFactory() {}

public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
Map<String, Codec> codecs = new HashMap<>();
codecs.put(DEFAULT_CODEC, new Composite103Codec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new Composite103Codec(Lucene103Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new Composite103Codec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZLIB, new Composite103Codec(Lucene103Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(DEFAULT_CODEC, new Composite104Codec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new Composite104Codec(Lucene104Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new Composite104Codec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZLIB, new Composite104Codec(Lucene104Codec.Mode.BEST_COMPRESSION, mapperService, logger));
return codecs;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@
*/

/**
* Responsible for handling all composite index codecs and operations associated with Composite100 codec
* Responsible for handling all composite index codecs and operations associated with Composite101 codec
*/
package org.opensearch.index.codec.composite.backward_codecs.composite101;
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,13 @@
* compatible open source license.
*/

package org.opensearch.index.codec.composite.composite103;
package org.opensearch.index.codec.composite.backward_codecs.composite103;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.backward_codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
import org.opensearch.index.mapper.MapperService;

Expand All @@ -34,10 +32,6 @@ public Composite103Codec() {
this(COMPOSITE_INDEX_CODEC_NAME, new Lucene103Codec(), null);
}

public Composite103Codec(Lucene103Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
}

/**
* Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
* this ctor.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

/**
* Responsible for handling all composite index codecs and operations associated with Composite103 codec
*/
package org.opensearch.index.codec.composite.backward_codecs.composite103;
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.codec.composite.composite104;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
import org.opensearch.index.mapper.MapperService;

/**
* Extends the Codec to support new file formats for composite indices eg: star tree index
* based on the mappings.
*
* @opensearch.experimental
*/
@ExperimentalApi
public class Composite104Codec extends FilterCodec {
public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite104Codec";
private final MapperService mapperService;

// needed for SPI - this is used in reader path
public Composite104Codec() {
this(COMPOSITE_INDEX_CODEC_NAME, new Lucene104Codec(), null);
}

public Composite104Codec(Lucene104Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
}

/**
* Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
* this ctor.
*
* @param name name of the codec
* @param delegate codec delegate
* @param mapperService mapper service instance
*/
protected Composite104Codec(String name, Codec delegate, MapperService mapperService) {
super(name, delegate);
this.mapperService = mapperService;
}

@Override
public DocValuesFormat docValuesFormat() {
return new Composite912DocValuesFormat(mapperService);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@
*/

/**
* Responsible for handling all composite index codecs and operations associated with Composite100 codec
* Responsible for handling all composite index codecs and operations associated with Composite104 codec
*/
package org.opensearch.index.codec.composite.composite103;
package org.opensearch.index.codec.composite.composite104;
Loading
Loading