diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java index 4b6b54979509c..17795bfdc755c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java @@ -26,7 +26,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; @@ -227,50 +226,44 @@ private static BlockLoader blockLoader(String name) { break; } ft.freeze(); - return new KeywordFieldMapper.KeywordFieldType( - w.name, - ft, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(ft.docValuesType() != DocValuesType.NONE), - syntheticSource - ).blockLoader(new MappedFieldType.BlockLoaderContext() { - @Override - public String indexName() { - return "benchmark"; - } + return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader( + new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "benchmark"; + } - @Override - public IndexSettings indexSettings() { - throw new UnsupportedOperationException(); - } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } - @Override - public MappedFieldType.FieldExtractPreference fieldExtractPreference() { - return MappedFieldType.FieldExtractPreference.NONE; - } + @Override + public MappedFieldType.FieldExtractPreference fieldExtractPreference() { + return MappedFieldType.FieldExtractPreference.NONE; + } - @Override - public SearchLookup lookup() { - throw new UnsupportedOperationException(); - } + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } - @Override - public Set sourcePaths(String name) { - return Set.of(name); - } + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } - @Override - public String parentField(String field) { - throw new UnsupportedOperationException(); - } + @Override + public String parentField(String field) { + throw new UnsupportedOperationException(); + } - @Override - public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { - return FieldNamesFieldMapper.FieldNamesFieldType.get(true); + @Override + public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { + return FieldNamesFieldMapper.FieldNamesFieldType.get(true); + } } - }); + ); } throw new IllegalArgumentException("can't read [" + name + "]"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d46761cbaa5b5..bb78f06684bf7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -722,7 +722,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } long scaledValue = encode(doubleValue, scalingFactor); - NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored); + NumberFieldMapper.NumberType.LONG.addFields( + context.doc(), + fieldType().name(), + scaledValue, + IndexType.points(indexed, hasDocValues), + stored + ); if (shouldStoreOffsets) { context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index 07bd713799c2a..0d77582bd9832 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -164,7 +164,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio tokenCount = countPositions(analyzer, fullPath(), value, enablePositionIncrements); } - NumberFieldMapper.NumberType.INTEGER.addFields(context.doc(), fieldType().name(), tokenCount, index, hasDocValues, store); + NumberFieldMapper.NumberType.INTEGER.addFields( + context.doc(), + fieldType().name(), + tokenCount, + IndexType.points(index, hasDocValues), + store + ); } /** diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 42ee978239294..036cb55fa03cf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappingParserContext; @@ -295,9 +296,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -344,9 +344,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index ff321303b56c0..1268705436d28 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -79,6 +79,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DocumentParserContext; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -301,7 +302,7 @@ public void testDuel() throws Exception { document.add(new TextField(entry.getKey(), value, Field.Store.NO)); } for (Integer intValue : intValues) { - NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, true, true, false); + NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, IndexType.points(true, true), false); } MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer()); duelRun(queryStore, memoryIndex, shardSearcher); @@ -424,7 +425,13 @@ public void testDuel2() throws Exception { } for (int[] range : ranges) { - NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", between(range[0], range[1]), true, true, false); + NumberFieldMapper.NumberType.INTEGER.addFields( + document, + "int_field", + between(range[0], range[1]), + IndexType.points(true, true), + false + ); logger.info("Test with document: {}" + document); MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer()); duelRun(queryStore, memoryIndex, shardSearcher); diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index ccd0b16666a6a..b8a0be5b6629e 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -104,7 +104,7 @@ public void postParse(DocumentParserContext context) { return; } final int value = context.sourceToParse().source().length(); - NumberType.INTEGER.addFields(context.doc(), fullPath(), value, true, true, true); + NumberType.INTEGER.addFields(context.doc(), fullPath(), value, IndexType.points(true, true), true); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 1352c40890fc4..97d075abeaa17 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -192,6 +192,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1); public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1); public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID = def(9_044_0_00, Version.LUCENE_10_3_1); + public static final IndexVersion TIME_SERIES_DIMENSIONS_USE_SKIPPERS = def(9_045_0_00, Version.LUCENE_10_3_1); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index d59c6c31deafa..6e95417116162 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -79,7 +79,7 @@ private static BooleanFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); private final Parameter> ignoreMalformed; private final Parameter nullValue = new Parameter<>( @@ -117,20 +117,9 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe IGNORE_MALFORMED_SETTING.get(indexSettings.getSettings()) ); this.script.precludesParameters(ignoreMalformed, nullValue); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension(), docValues::get); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension); addScriptValidation(script, indexed, docValues); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { - if (v && (indexed.getValue() == false || docValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + docValues.name - + "] are true" - ); - } - }); } public Builder dimension(boolean dimension) { @@ -152,6 +141,21 @@ protected Parameter[] getParameters() { dimension }; } + private IndexType indexType() { + if (indexed.get() && indexSettings.getIndexVersionCreated().isLegacyIndexVersion() == false) { + return IndexType.terms(true, docValues.getValue()); + } + if (docValues.get() == false) { + return IndexType.NONE; + } + if (dimension.get() + && indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } + return IndexType.docValuesOnly(); + } + @Override public BooleanFieldMapper build(MapperBuilderContext context) { if (inheritDimensionParameterFromParentObject(context)) { @@ -159,9 +163,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) { } MappedFieldType ft = new BooleanFieldType( context.buildFullName(leafName()), - indexed.getValue() && indexSettings.getIndexVersionCreated().isLegacyIndexVersion() == false, + indexType(), stored.getValue(), - docValues.getValue(), nullValue.getValue(), scriptValues(), meta.getValue(), @@ -215,16 +218,15 @@ public static final class BooleanFieldType extends TermBasedFieldType { public BooleanFieldType( String name, - boolean isIndexed, + IndexType indexType, boolean isStored, - boolean hasDocValues, Boolean nullValue, FieldValues scriptValues, Map meta, boolean isDimension, boolean isSyntheticSource ) { - super(name, IndexType.terms(isIndexed, hasDocValues), isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, indexType, isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.nullValue = nullValue; this.scriptValues = scriptValues; this.isDimension = isDimension; @@ -232,15 +234,11 @@ public BooleanFieldType( } public BooleanFieldType(String name) { - this(name, true); - } - - public BooleanFieldType(String name, boolean isIndexed) { - this(name, isIndexed, true); + this(name, IndexType.terms(true, true)); } - public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) { - this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false, false); + public BooleanFieldType(String name, IndexType indexType) { + this(name, indexType, true, false, null, Collections.emptyMap(), false, false); } @Override @@ -601,7 +599,11 @@ private void indexValue(DocumentParserContext context, Boolean value) { context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); } if (hasDocValues) { - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + if (fieldType().indexType.hasDocValuesSkipper()) { + context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), value ? 1 : 0)); + } else { + context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + } } else { context.addToFieldNames(fieldType().name()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index dd35576299a0e..412c7648f2acf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1314,6 +1314,21 @@ public static Parameter indexParam(Function initi return Parameter.boolParam("index", false, initializer, defaultValue); } + public static Parameter indexParam( + Function initializer, + IndexSettings indexSettings, + Supplier isDimension + ) { + return Parameter.boolParam( + "index", + false, + initializer, + () -> isDimension.get() == false + || indexSettings.useDocValuesSkipper() == false + || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS) + ); + } + public static Parameter storeParam(Function initializer, boolean defaultValue) { return Parameter.boolParam("store", false, initializer, defaultValue); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 655c25924d887..56ec1e796f352 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -143,7 +143,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe } }); // We allow `time_series_dimension` parameter to be parsed, but only allow it to be `false` - this.dimension = TimeSeriesParams.dimensionParam(m -> false).addValidator(v -> { + this.dimension = TimeSeriesParams.dimensionParam(m -> false, () -> true).addValidator(v -> { if (v) { throw new IllegalArgumentException( "Parameter [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] cannot be set to geo_point" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java index 4759144e8f949..0bf9877297f8a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexType.java @@ -9,6 +9,11 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; + import java.util.Objects; /** @@ -118,6 +123,25 @@ public static IndexType terms(boolean isIndexed, boolean hasDocValues) { return new IndexType(isIndexed, false, false, false, hasDocValues, false); } + /** + * @return a terms-based IndexType from a lucene FieldType + */ + public static IndexType terms(FieldType fieldType) { + if (fieldType.indexOptions() == IndexOptions.NONE) { + if (fieldType.docValuesType() == DocValuesType.NONE) { + return NONE; + } + if (fieldType.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE) { + return docValuesOnly(); + } + return skippers(); + } + if (fieldType.docValuesType() == DocValuesType.NONE) { + return terms(true, false); + } + return terms(true, true); + } + /** * @return an IndexType with docValuesSkippers */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index deea53d180e10..4bd1a97e9dbe1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -74,7 +74,7 @@ private static IpFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); @@ -105,20 +105,9 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe IGNORE_MALFORMED_SETTING.get(indexSettings.getSettings()) ); this.script.precludesParameters(nullValue, ignoreMalformed); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension); addScriptValidation(script, indexed, hasDocValues); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + hasDocValues.name - + "] are true" - ); - } - }); } Builder nullValue(String nullValue) { @@ -186,6 +175,11 @@ private IndexType indexType() { if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) { return hasDocValues.get() ? IndexType.archivedPoints() : IndexType.NONE; } + if (dimension.get() + && indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } return IndexType.points(indexed.get(), hasDocValues.get()); } @@ -650,7 +644,11 @@ private void indexValue(DocumentParserContext context, ESInetAddressPoint addres doc.add(address); } if (hasDocValues) { - doc.add(new SortedSetDocValuesField(fieldType().name(), address.binaryValue())); + if (indexed == false) { + doc.add(SortedSetDocValuesField.indexedField(fieldType().name(), address.binaryValue())); + } else { + doc.add(new SortedSetDocValuesField(fieldType().name(), address.binaryValue())); + } } else if (stored || indexed) { context.addToFieldNames(fieldType().name()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7296aad9fd292..4bca305eb5da9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -171,7 +171,7 @@ private static KeywordFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); @@ -255,21 +255,12 @@ public Builder( ); this.script.precludesParameters(nullValue); + + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension(), hasDocValues::get) + .precludesParameters(normalizer); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension); addScriptValidation(script, indexed, hasDocValues); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + hasDocValues.name - + "] are true" - ); - } - }).precludesParameters(normalizer); this.ignoreAbove = Parameter.ignoreAboveParam( m -> toType(m).fieldType().ignoreAbove().get(), IGNORE_ABOVE_SETTING.get(indexSettings.getSettings()) @@ -323,6 +314,10 @@ public boolean hasDocValues() { return this.hasDocValues.get(); } + public SimilarityProvider similarity() { + return this.similarity.get(); + } + public Builder dimension(boolean dimension) { this.dimension.setValue(dimension); return this; @@ -405,10 +400,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } return new KeywordFieldType( context.buildFullName(leafName()), - fieldType, + IndexType.terms(fieldType), + new TextSearchInfo(fieldType, similarity.get(), searchAnalyzer, quoteAnalyzer), normalizer, - searchAnalyzer, - quoteAnalyzer, this, context.isSourceSynthetic() ); @@ -416,26 +410,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = resolveFieldType( - forceDocValuesSkipper, - hasDocValues.get(), - indexSettings, - context.buildFullName(leafName()) - ); - fieldtype.setOmitNorms(this.hasNorms.getValue() == false); - fieldtype.setStored(this.stored.getValue()); - fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); - if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { - // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) - fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); - } - if (fieldtype.equals(Defaults.FIELD_TYPE)) { - // deduplicate in the common default case to save some memory - fieldtype = Defaults.FIELD_TYPE; - } - if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { - fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; - } + FieldType fieldtype = resolveFieldType(forceDocValuesSkipper, context.buildFullName(leafName())); super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); @@ -458,24 +433,38 @@ public KeywordFieldMapper build(MapperBuilderContext context) { ); } - private static FieldType resolveFieldType( - final boolean forceDocValuesSkipper, - boolean hasDocValues, - final IndexSettings indexSettings, - final String fullFieldName - ) { - if (forceDocValuesSkipper || shouldUseDocValuesSkipper(hasDocValues, indexSettings, fullFieldName)) { - return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + private FieldType resolveFieldType(final boolean forceDocValuesSkipper, final String fullFieldName) { + FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); + if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseDimensionSkipper()) { + fieldtype = new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + } + fieldtype.setOmitNorms(this.hasNorms.getValue() == false); + fieldtype.setStored(this.stored.getValue()); + fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { + // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) + fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); + } + if (fieldtype.equals(Defaults.FIELD_TYPE)) { + // deduplicate in the common default case to save some memory + fieldtype = Defaults.FIELD_TYPE; + } + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { + fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; } - return new FieldType(Defaults.FIELD_TYPE); + return fieldtype; } - private static boolean shouldUseDocValuesSkipper( - final boolean hasDocValues, - final IndexSettings indexSettings, - final String fullFieldName - ) { - return hasDocValues + private boolean shouldUseDimensionSkipper() { + return hasDocValues.get() + && dimension.get() + && indexed.get() == false + && indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + } + + private boolean shouldUseHostnameSkipper(final String fullFieldName) { + return hasDocValues.get() && indexSettings.useDocValuesSkipper() && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) && IndexMode.LOGSDB.equals(indexSettings.getMode()) @@ -500,25 +489,20 @@ public static final class KeywordFieldType extends TextFamilyFieldType { private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; - private final boolean hasDocValuesSkipper; public KeywordFieldType( String name, - FieldType fieldType, + IndexType indexType, + TextSearchInfo textSearchInfo, NamedAnalyzer normalizer, - NamedAnalyzer searchAnalyzer, - NamedAnalyzer quoteAnalyzer, Builder builder, boolean isSyntheticSource ) { super( name, - IndexType.terms( - fieldType.indexOptions() != IndexOptions.NONE && builder.indexCreatedVersion.isLegacyIndexVersion() == false, - builder.hasDocValues.get() - ), - fieldType.stored(), - textSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, quoteAnalyzer), + indexType, + builder.stored.get(), + textSearchInfo, builder.meta.getValue(), isSyntheticSource, builder.isWithinMultiField @@ -533,7 +517,6 @@ public KeywordFieldType( this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); - this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name) { @@ -548,17 +531,16 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.hasDocValuesSkipper = false; } - public KeywordFieldType(String name, FieldType fieldType) { + public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSource) { super( name, - IndexType.terms(fieldType.indexOptions() != IndexOptions.NONE, false), - false, + IndexType.terms(fieldType), + fieldType.stored(), textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap(), - false, + isSyntheticSource, false ); this.normalizer = Lucene.KEYWORD_ANALYZER; @@ -567,7 +549,6 @@ public KeywordFieldType(String name, FieldType fieldType) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -586,7 +567,6 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.hasDocValuesSkipper = false; } @Override @@ -1051,10 +1031,6 @@ public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } - public boolean hasDocValuesSkipper() { - return hasDocValuesSkipper; - } - @Override public Query automatonQuery( Supplier automatonSupplier, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 434e74fefb90a..cfbab8887bdc7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -166,7 +166,12 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti XContentBuilder::field, Objects::toString ).acceptsNull(); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { + if (dimension.get()) { + return indexSettings.useDocValuesSkipper() == false + || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + } if (indexSettings.getMode() == IndexMode.TIME_SERIES) { var metricType = getMetric().getValue(); return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; @@ -174,19 +179,6 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti return true; } }); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + hasDocValues.name - + "] are true" - ); - } - }); this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> { if (v != null && hasDocValues.getValue() == false) { @@ -224,6 +216,13 @@ private IndexType indexType() { if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) { return IndexType.archivedPoints(); } + if (indexSettings.useDocValuesSkipper() + && indexed.get() == false + && hasDocValues.get() + && dimension.get() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } return IndexType.points(indexed.get(), hasDocValues.get()); } @@ -413,13 +412,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final float f = value.floatValue(); - if (indexed) { + if (indexType.hasPoints()) { document.add(new HalfFloatPoint(name, f)); } - if (docValued) { - document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + } else { + document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + } } if (stored) { document.add(new StoredField(name, f)); @@ -606,13 +609,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final float f = value.floatValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new FloatField(name, f, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f))); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(f))); + } else { + document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f))); + } + } else if (indexType.hasPoints()) { document.add(new FloatPoint(name, f)); } if (stored) { @@ -766,13 +773,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final double d = value.doubleValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new DoubleField(name, d, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d))); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(d))); + } else { + document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d))); + } + } else if (indexType.hasPoints()) { document.add(new DoublePoint(name, d)); } if (stored) { @@ -905,8 +916,8 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { - INTEGER.addFields(document, name, value, indexed, docValued, stored); + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { + INTEGER.addFields(document, name, value, indexType, stored); } @Override @@ -1033,8 +1044,8 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { - INTEGER.addFields(document, name, value, indexed, docValued, stored); + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { + INTEGER.addFields(document, name, value, indexType, stored); } @Override @@ -1230,13 +1241,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final int i = value.intValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new IntField(name, i, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, i)); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, i)); + } else { + document.add(new SortedNumericDocValuesField(name, i)); + } + } else if (indexType.hasPoints()) { document.add(new IntPoint(name, i)); } if (stored) { @@ -1392,13 +1407,17 @@ public Query rangeQuery( } @Override - public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) { + public void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored) { final long l = value.longValue(); - if (indexed && docValued) { + if (indexType.hasPoints() && indexType.hasDocValues()) { document.add(new LongField(name, l, Field.Store.NO)); - } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, l)); - } else if (indexed) { + } else if (indexType.hasDocValues()) { + if (indexType.hasDocValuesSkipper()) { + document.add(SortedNumericDocValuesField.indexedField(name, l)); + } else { + document.add(new SortedNumericDocValuesField(name, l)); + } + } else if (indexType.hasPoints()) { document.add(new LongPoint(name, l)); } if (stored) { @@ -1545,18 +1564,10 @@ public abstract Query rangeQuery( * @param document document to add fields to * @param name field name * @param value value to map - * @param indexed whether or not the field is indexed - * @param docValued whether or not doc values should be added + * @param indexType an IndexType describing the index structures to be added * @param stored whether or not the field is stored */ - public abstract void addFields( - LuceneDocument document, - String name, - Number value, - boolean indexed, - boolean docValued, - boolean stored - ); + public abstract void addFields(LuceneDocument document, String name, Number value, IndexType indexType, boolean stored); /** * For a given {@code Number}, returns the sortable long representation that will be stored in the doc values. @@ -2268,7 +2279,7 @@ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { context.getRoutingFields().addLong(fieldType().name(), numericValue.longValue()); } - fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); + fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, fieldType().indexType, stored); if (false == allowMultipleValues && (indexed || hasDocValues || stored)) { // the last field is the current field, Add to the key map, so that we can validate if it has been added diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java index 1f4676d60814f..66af7bb97f59c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.Locale; +import java.util.function.BooleanSupplier; import java.util.function.Function; /** @@ -88,8 +89,14 @@ public static FieldMapper.Parameter metricParam(Function dimensionParam(Function initializer) { - return FieldMapper.Parameter.boolParam(TIME_SERIES_DIMENSION_PARAM, false, initializer, false); + public static FieldMapper.Parameter dimensionParam(Function initializer, BooleanSupplier hasDocValues) { + return FieldMapper.Parameter.boolParam(TIME_SERIES_DIMENSION_PARAM, false, initializer, false).addValidator(v -> { + if (v && (hasDocValues.getAsBoolean() == false)) { + throw new IllegalArgumentException( + "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] requires that [doc_values] is true" + ); + } + }); } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 5431e3925a1d7..4fd688557025f 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -341,7 +341,16 @@ public void testRequireDocValuesOnDoubles() { public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); doTestRequireDocValues( - new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false, false) + new BooleanFieldMapper.BooleanFieldType( + "field", + IndexType.terms(true, false), + false, + null, + null, + Collections.emptyMap(), + false, + false + ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java index 02a9d7dc59940..79e16220ecc57 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues.Doubles; import org.elasticsearch.index.fielddata.ScriptDocValues.DoublesSupplier; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.script.field.DelegateDocValuesField; @@ -34,7 +35,7 @@ public void testSingleValued() throws IOException { // we need the default codec to check for singletons IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec())); LuceneDocument doc = new LuceneDocument(); - NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, false, true, false); + NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, IndexType.points(false, true), false); w.addDocument(doc); final DirectoryReader dirReader = DirectoryReader.open(w); LeafReader reader = getOnlyLeafReader(dirReader); @@ -54,8 +55,8 @@ public void testMultiValued() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); LuceneDocument doc = new LuceneDocument(); - NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, false, true, false); - NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 2f, false, true, false); + NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 3f, IndexType.points(false, true), false); + NumberFieldMapper.NumberType.HALF_FLOAT.addFields(doc, "half_float", 2f, IndexType.points(false, true), false); w.addDocument(doc); final DirectoryReader dirReader = DirectoryReader.open(w); LeafReader reader = getOnlyLeafReader(dirReader); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 76df9b6303109..9be4ff52d39f4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -226,6 +226,8 @@ public void testDimension() throws IOException { assertDimension(true, BooleanFieldMapper.BooleanFieldType::isDimension); assertDimension(false, BooleanFieldMapper.BooleanFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -234,30 +236,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index 38f73cd9f681d..3c521b7e55caf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -40,11 +40,11 @@ public void testTermQuery() { assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", MOCK_CONTEXT)); assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", MOCK_CONTEXT)); - MappedFieldType ft2 = new BooleanFieldMapper.BooleanFieldType("field", false); + MappedFieldType ft2 = new BooleanFieldMapper.BooleanFieldType("field", IndexType.terms(false, true)); assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), ft2.termQuery("true", MOCK_CONTEXT)); assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 0), ft2.termQuery("false", MOCK_CONTEXT)); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", IndexType.NONE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", MOCK_CONTEXT)); assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); } @@ -54,11 +54,11 @@ public void testRangeQuery() { Query expected = new TermRangeQuery("field", BooleanFieldMapper.Values.FALSE, BooleanFieldMapper.Values.TRUE, true, true); assertEquals(expected, ft.rangeQuery("false", "true", true, true, null, null, null, MOCK_CONTEXT)); - ft = new BooleanFieldMapper.BooleanFieldType("field", false); + ft = new BooleanFieldMapper.BooleanFieldType("field", IndexType.points(false, true)); expected = SortedNumericDocValuesField.newSlowRangeQuery("field", 0, 1); assertEquals(expected, ft.rangeQuery("false", "true", true, true, null, null, null, MOCK_CONTEXT)); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", IndexType.NONE); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> unsearchable.rangeQuery("false", "true", true, true, null, null, null, MOCK_CONTEXT) @@ -75,10 +75,9 @@ public void testFetchSourceValue() throws IOException { MappedFieldType nullFieldType = new BooleanFieldMapper.BooleanFieldType( "field", - true, + IndexType.points(true, true), false, true, - true, null, Collections.emptyMap(), false, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index e0308b5fd8bbf..4874442b9c5a0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -224,6 +224,8 @@ public void testDimension() throws IOException { assertDimension(true, IpFieldMapper.IpFieldType::isDimension); assertDimension(false, IpFieldMapper.IpFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -232,30 +234,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index e2b3de2672654..5d15ce93737eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -427,6 +427,8 @@ public void testDimension() throws IOException { assertDimension(true, KeywordFieldMapper.KeywordFieldType::isDimension); assertDimension(false, KeywordFieldMapper.KeywordFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionAndIgnoreAbove() throws IOException { @@ -455,30 +457,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } @@ -983,9 +969,9 @@ public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); if (mapperService.getIndexSettings().useDocValuesSkipper()) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1004,7 +990,7 @@ public void testFieldTypeDefault_StandardMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().indexType().hasTerms()); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); } public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { @@ -1021,7 +1007,7 @@ public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().indexType().hasTerms()); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType.hasDocValuesSkipper()); } public void testFieldTypeDefault_ConfiguredIndexedWithSettingOverride() throws IOException { @@ -1039,9 +1025,9 @@ public void testFieldTypeDefault_ConfiguredIndexedWithSettingOverride() throws I final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (mapperService.getIndexSettings().useDocValuesSkipper()) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1061,9 +1047,9 @@ public void testFieldTypeDefault_ConfiguredIndexedWithoutSettingOverride() throw final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (mapperService.getIndexSettings().useDocValuesSkipper()) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1083,9 +1069,9 @@ public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (mapperService.getIndexSettings().useDocValuesSkipper()) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasTerms()); } } @@ -1104,7 +1090,7 @@ public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().indexType().hasTerms()); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); } public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { @@ -1123,9 +1109,9 @@ public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IO final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); if (mapperService.getIndexSettings().useDocValuesSkipper()) { - assertTrue(mapper.fieldType().hasDocValuesSkipper()); + assertTrue(mapper.fieldType().indexType().hasDocValuesSkipper()); } else { - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); assertTrue(mapper.fieldType().indexType().hasOnlyDocValues()); } } @@ -1146,7 +1132,7 @@ public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValues()); assertThat(mapper.fieldType().indexType(), equalTo(IndexType.NONE)); - assertFalse(mapper.fieldType().hasDocValuesSkipper()); + assertFalse(mapper.fieldType().indexType().hasDocValuesSkipper()); } public void testValueIsStoredWhenItExceedsIgnoreAboveAndFieldIsNotAMultiField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 8e2e29343a195..8c2f19d02e046 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -143,7 +143,7 @@ public void testExistsQuery() { { FieldType fieldType = new FieldType(); fieldType.setOmitNorms(false); - KeywordFieldType ft = new KeywordFieldType("field", fieldType); + KeywordFieldType ft = new KeywordFieldType("field", fieldType, false); // updated in #130531 so that a field that is neither indexed nor has doc values will generate a TermQuery // to avoid ISE from FieldExistsQuery assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(MOCK_CONTEXT)); @@ -327,9 +327,8 @@ public void testIgnoreAboveIndexLevelSetting() { KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -359,9 +358,8 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsGiven() { KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -390,9 +388,8 @@ public void testIgnoreAboveIsSetReturnsFalseWhenIgnoreAboveIsNotGiven() { KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -422,9 +419,8 @@ public void testIgnoreAboveIsSetReturnsFalseWhenIgnoreAboveIsGivenButItsTheSameA KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -454,9 +450,8 @@ public void testIgnoreAboveIsSetReturnsFalseWhenIgnoreAboveIsGivenButItsTheSameA KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -486,9 +481,8 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsGivenAsLogsdbDefault KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -518,9 +512,8 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsConfiguredAtIndexLev KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -534,7 +527,7 @@ public void testIgnoreAboveIsSetReturnsTrueWhenIgnoreAboveIsConfiguredAtIndexLev public void testIgnoreAboveIsSetReturnsFalseForNonPrimaryConstructor() { // given KeywordFieldType fieldType1 = new KeywordFieldType("field"); - KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class)); + KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class), false); KeywordFieldType fieldType3 = new KeywordFieldType("field", true, true, Collections.emptyMap()); KeywordFieldType fieldType4 = new KeywordFieldType("field", mock(NamedAnalyzer.class)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index c40e720360c2a..1fca19f88384a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -679,7 +679,7 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier valueSu final int numDocs = TestUtil.nextInt(random(), 100, 500); for (int i = 0; i < numDocs; ++i) { final LuceneDocument doc = new LuceneDocument(); - type.addFields(doc, "foo", valueSupplier.get(), true, true, false); + type.addFields(doc, "foo", valueSupplier.get(), IndexType.points(true, true), false); w.addDocument(doc); } DirectoryReader reader = DirectoryReader.open(w); @@ -733,7 +733,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS final int numDocs = TestUtil.nextInt(random(), 100, 500); for (int i = 0; i < numDocs; ++i) { final LuceneDocument doc = new LuceneDocument(); - type.addFields(doc, "field", valueSupplier.get(), true, true, false); + type.addFields(doc, "field", valueSupplier.get(), IndexType.points(true, true), false); w.addDocument(doc); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 4787b444d732f..95921cd410368 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -353,9 +353,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true @@ -404,9 +403,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), + IndexType.terms(true, true), + new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)), mock(NamedAnalyzer.class), builder, true diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 0af4a8b31cd76..69cfa0bcd240f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -1486,7 +1486,7 @@ public void testDocValuesFieldExistsForNumber() throws IOException { NumberFieldMapper.NumberFieldType ft = new NumberFieldMapper.NumberFieldType("f", numberType); docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, true, i -> { final LuceneDocument document = new LuceneDocument(); - numberType.addFields(document, "f", i, true, true, false); + numberType.addFields(document, "f", i, IndexType.points(true, true), false); return document; }); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index d58b72f788c03..b71cf00859700 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -235,7 +235,7 @@ public void testFloatRangesExclusiveEndpoint() throws IOException { MappedFieldType field = new NumberFieldMapper.NumberFieldType(fieldName, NumberType.FLOAT); testCase(iw -> { LuceneDocument doc = new LuceneDocument(); - NumberType.FLOAT.addFields(doc, fieldName, 0.04F, false, true, false); + NumberType.FLOAT.addFields(doc, fieldName, 0.04F, IndexType.points(false, true), false); iw.addDocument(doc); }, result -> { InternalRange range = (InternalRange) result; @@ -259,7 +259,7 @@ public void testHalfFloatRangesExclusiveEndpoint() throws IOException { MappedFieldType field = new NumberFieldMapper.NumberFieldType(fieldName, NumberType.HALF_FLOAT); testCase(iw -> { LuceneDocument doc = new LuceneDocument(); - NumberType.HALF_FLOAT.addFields(doc, fieldName, 0.0152F, false, true, false); + NumberType.HALF_FLOAT.addFields(doc, fieldName, 0.0152F, IndexType.points(false, true), false); iw.addDocument(doc); }, result -> { InternalRange range = (InternalRange) result; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index 504b757ffa42f..f6806111050ae 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.BinaryFieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; @@ -259,13 +260,13 @@ public void testNumericSignificance() throws IOException { for (int i = 0; i < 10; i++) { LuceneDocument doc = new LuceneDocument(); if (i % 2 == 0) { - NumberType.LONG.addFields(doc, "long_field", ODD_VALUE, true, true, false); + NumberType.LONG.addFields(doc, "long_field", ODD_VALUE, IndexType.points(true, true), false); doc.add(new Field("text", "odd", TextFieldMapper.Defaults.FIELD_TYPE)); } else { - NumberType.LONG.addFields(doc, "long_field", EVEN_VALUE, true, true, false); + NumberType.LONG.addFields(doc, "long_field", EVEN_VALUE, IndexType.points(true, true), false); doc.add(new Field("text", "even", TextFieldMapper.Defaults.FIELD_TYPE)); } - NumberType.LONG.addFields(doc, "long_field", COMMON_VALUE, true, true, false); + NumberType.LONG.addFields(doc, "long_field", COMMON_VALUE, IndexType.points(true, true), false); w.addDocument(doc); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 9d4468161219d..ed8b7035a7252 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -12,6 +12,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriterConfig; @@ -389,6 +390,32 @@ protected void assertDimension(boolean isDimension, Function che @SuppressWarnings("unchecked") // Syntactic sugar in tests T fieldType = (T) mapperService.fieldType("field"); assertThat(checker.apply(fieldType), equalTo(isDimension)); + + Settings settings = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build(); + mapperService = createMapperService(settings, fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", isDimension); + })); + + assertThat( + mapperService.fieldType("field").indexType().hasDocValuesSkipper(), + equalTo(mapperService.getIndexSettings().useDocValuesSkipper() && isDimension) + ); + } + + public void assertDimensionIndexing() throws IOException { + Settings settings = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build(); + MapperService mapperService = createMapperService(settings, fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + })); + assumeTrue("Skippers disabled by feature flag", mapperService.getIndexSettings().useDocValuesSkipper()); + + ParsedDocument doc = mapperService.documentMapper().parse(source(this::writeField)); + IndexableField field = doc.rootDoc().getField("field"); + assertSame(DocValuesSkipIndexType.RANGE, field.fieldType().docValuesSkipIndexType()); + assertSame(IndexOptions.NONE, field.fieldType().indexOptions()); + assertEquals(0, field.fieldType().pointDimensionCount()); } protected void assertMetricType(String metricType, Function> checker) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 2f16534908b2e..310d891947fb2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -254,6 +254,8 @@ public void testDimension() throws IOException { // dimension = true is allowed assertDimension(true, NumberFieldMapper.NumberFieldType::isDimension); + + assertDimensionIndexing(); } public void testMetricType() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index 64a7868607cf1..42f757e73018c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -39,6 +39,8 @@ public void testDimension() throws IOException { assertDimension(true, NumberFieldMapper.NumberFieldType::isDimension); assertDimension(false, NumberFieldMapper.NumberFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -47,30 +49,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 7a0e5c1c6cd83..dca14c8dd0d2c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; @@ -1401,15 +1400,7 @@ private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { ft.setDocValuesType(DocValuesType.NONE); ft.setStored(true); ft.freeze(); - return new KeywordFieldMapper.KeywordFieldType( - name, - ft, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(false), - true // TODO randomize - load from stored keyword fields if stored even in synthetic source - ); + return new KeywordFieldMapper.KeywordFieldType(name, ft, false); } @AwaitsFix(bugUrl = "Get working for multiple indices") diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 9ee061b6e1d6e..cf4132fb52180 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -1577,15 +1577,7 @@ private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { ft.setDocValuesType(DocValuesType.NONE); ft.setStored(true); ft.freeze(); - return new KeywordFieldMapper.KeywordFieldType( - name, - ft, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, - new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(false), - true // TODO randomize - load from stored keyword fields if stored even in synthetic source - ); + return new KeywordFieldMapper.KeywordFieldType(name, ft, false); } private TextFieldMapper.TextFieldType storedTextField(String name) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index f15c831acf5a8..e708eae794be7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -41,10 +41,12 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -242,9 +244,8 @@ static MappedFieldType createUnmappedFieldType(String name, DefaultShardContext builder.indexed(false); return new KeywordFieldMapper.KeywordFieldType( name, - UNMAPPED_FIELD_TYPE, - Lucene.KEYWORD_ANALYZER, - Lucene.KEYWORD_ANALYZER, + IndexType.terms(false, false), + new TextSearchInfo(UNMAPPED_FIELD_TYPE, builder.similarity(), Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Lucene.KEYWORD_ANALYZER, builder, context.ctx.isSourceSynthetic() diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index cc8b11e7eedb2..59da232924c09 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -123,7 +123,12 @@ public Builder(String name, IndexSettings indexSettings) { XContentBuilder::field, Objects::toString ).acceptsNull(); + this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get); this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { + if (dimension.get()) { + return indexSettings.useDocValuesSkipper() == false + || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); + } if (indexSettings.getMode() == IndexMode.TIME_SERIES) { var metricType = getMetric().getValue(); return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; @@ -131,19 +136,6 @@ public Builder(String name, IndexSettings indexSettings) { return true; } }); - this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { - throw new IllegalArgumentException( - "Field [" - + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM - + "] requires that [" - + indexed.name - + "] and [" - + hasDocValues.name - + "] are true" - ); - } - }); this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> { if (v != null && hasDocValues.getValue() == false) { @@ -198,6 +190,17 @@ Number parsedNullValue() { return parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); } + private IndexType indexType() { + if (indexSettings.useDocValuesSkipper() + && indexed.get() == false + && hasDocValues.get() + && dimension.get() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) { + return IndexType.skippers(); + } + return IndexType.points(indexed.get(), hasDocValues.get()); + } + @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { if (inheritDimensionParameterFromParentObject(context)) { @@ -205,7 +208,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( context.buildFullName(leafName()), - IndexType.points(indexed.get(), hasDocValues.get()), + indexType(), stored.getValue(), parsedNullValue(), meta.getValue(), @@ -767,7 +770,11 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (indexed && hasDocValues) { fields.add(new LongField(fieldType().name(), numericValue, Field.Store.NO)); } else if (hasDocValues) { - fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue)); + if (fieldType().indexType().hasDocValuesSkipper()) { + fields.add(SortedNumericDocValuesField.indexedField(fieldType().name(), numericValue)); + } else { + fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue)); + } } else if (indexed) { fields.add(new LongPoint(fieldType().name(), numericValue)); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index ea3e4df43d615..c852f3daedb39 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -228,6 +228,8 @@ public void testDimension() throws IOException { assertDimension(true, UnsignedLongFieldMapper.UnsignedLongFieldType::isDimension); assertDimension(false, UnsignedLongFieldMapper.UnsignedLongFieldType::isDimension); + + assertDimensionIndexing(); } public void testDimensionIndexedAndDocvalues() { @@ -236,30 +238,14 @@ public void testDimensionIndexedAndDocvalues() { minimalMapping(b); b.field("time_series_dimension", true).field("index", false).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true).field("index", true).field("doc_values", false); }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); - } - { - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true).field("index", false).field("doc_values", true); - }))); - assertThat( - e.getCause().getMessage(), - containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true") - ); + assertThat(e.getCause().getMessage(), containsString("Field [time_series_dimension] requires that [doc_values] is true")); } }