diff --git a/docs/changelog/135393.yaml b/docs/changelog/135393.yaml new file mode 100644 index 0000000000000..c80e101355878 --- /dev/null +++ b/docs/changelog/135393.yaml @@ -0,0 +1,5 @@ +pr: 135393 +summary: Improve block loader for source only runtime IP fields +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java new file mode 100644 index 0000000000000..1d837ea9daddb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + +public class IpFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader { + + public IpFallbackSyntheticSourceReader(Object nullValue) { + super(nullValue); + } + + @Override + public void convertValue(Object value, List accumulator) { + try { + if (value instanceof InetAddress ia) { + accumulator.add(ia); + } else { + InetAddress address = InetAddresses.forString(value.toString()); + accumulator.add(address); + } + } catch (Exception e) { + // value is malformed, skip it + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + BlockLoader.BytesRefBuilder builder = (BlockLoader.BytesRefBuilder) blockBuilder; + for (InetAddress address : values) { + var bytesRef = new BytesRef(InetAddressPoint.encode(address)); + builder.appendBytesRef(bytesRef); + } + } + + @Override + protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + try { + InetAddress address = InetAddresses.forString(parser.text()); + accumulator.add(address); + } catch (Exception e) { + // value is malformed, skip it + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 3781242815e95..781e4e27752a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -42,7 +42,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentString; import java.io.IOException; @@ -52,7 +51,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -480,44 +478,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { - var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { - @Override - public void convertValue(Object value, List accumulator) { - if (value instanceof InetAddress ia) { - accumulator.add(ia); - } - - try { - var address = InetAddresses.forString(value.toString()); - accumulator.add(address); - } catch (Exception e) { - // Malformed value, skip it - } - } - - @Override - protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { - // aligned with #parseCreateField() - String value = parser.text(); - - try { - var address = InetAddresses.forString(value); - accumulator.add(address); - } catch (Exception e) { - // Malformed value, skip it - } - } - - @Override - public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { - var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder; - - for (var value : values) { - bytesRefBuilder.appendBytesRef(new BytesRef(InetAddressPoint.encode(value))); - } - } - }; - + var reader = new IpFallbackSyntheticSourceReader(nullValue); return new FallbackSyntheticSourceBlockLoader( reader, name(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index 9923235f09025..8c8c3f004dcb1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -213,6 +213,16 @@ private Query cidrQuery(String term, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { + FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader = fallbackSyntheticSourceBlockLoader( + blContext, + BlockLoader.BlockFactory::bytesRefs, + () -> new IpFallbackSyntheticSourceReader(null) + ); + + if (fallbackSyntheticSourceBlockLoader != null) { + return fallbackSyntheticSourceBlockLoader; + } return new IpScriptBlockDocValuesReader.IpScriptBlockLoader(leafFactory(blContext.lookup())); } + } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index 7e9a236f6cc74..81e85d7447f7b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -30,6 +30,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.BinaryScriptFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues.Strings; @@ -43,20 +44,27 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; public class IpScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase { + private static final BytesRef EMPTY_IP = null; + private static final BytesRef MALFORMED_IP = null; + @Override protected ScriptFactory parseFromSource() { return IpFieldScript.PARSE_FROM_SOURCE; @@ -280,6 +288,141 @@ public void testBlockLoader() throws IOException { } } + public void testBlockLoaderSourceOnlyRuntimeField() throws IOException { + try ( + Directory directory = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE)) + ) { + // given + // try multiple variations of boolean as they're all encoded slightly differently + iw.addDocuments( + List.of( + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"192.168.0.1\"]}"))), + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"2001:db8::1\"]}"))), + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"\"]}"))), + // ensure a malformed value doesn't crash + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"potato\"]}"))) + ) + ); + IpScriptFieldType fieldType = simpleSourceOnlyMappedFieldType(); + List expected = Arrays.asList( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8::1"))), + EMPTY_IP, + MALFORMED_IP + ); + + try (DirectoryReader reader = iw.getReader()) { + // when + BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY, true)); + + // then + + // assert loader is of expected instance type + assertThat(loader, instanceOf(IpScriptBlockDocValuesReader.IpScriptBlockLoader.class)); + + // ignored source doesn't support column at a time loading: + var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst()); + assertThat(columnAtATimeLoader, instanceOf(IpScriptBlockDocValuesReader.class)); + + var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst()); + assertThat(rowStrideReader, instanceOf(IpScriptBlockDocValuesReader.class)); + + // assert values + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 0), equalTo(expected)); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected)); + } + } + } + + public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IOException { + try ( + Directory directory = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE)) + ) { + // given + // try multiple variations of boolean as they're all encoded slightly differently + iw.addDocuments( + List.of( + createDocumentWithIgnoredSource("[\"192.168.0.1\"]"), + createDocumentWithIgnoredSource("[\"2001:db8::1\"]"), + createDocumentWithIgnoredSource("[\"\"]"), + // ensure a malformed value doesn't crash + createDocumentWithIgnoredSource("[\"potato\"]") + ) + ); + + Settings settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build(); + IpScriptFieldType fieldType = simpleSourceOnlyMappedFieldType(); + List expected = Arrays.asList( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8::1"))), + EMPTY_IP, + MALFORMED_IP + ); + + try (DirectoryReader reader = iw.getReader()) { + // when + BlockLoader loader = fieldType.blockLoader(blContext(settings, true)); + + // then + + // assert loader is of expected instance type + assertThat(loader, instanceOf(FallbackSyntheticSourceBlockLoader.class)); + + // ignored source doesn't support column at a time loading: + var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst()); + assertThat(columnAtATimeLoader, nullValue()); + + var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst()); + assertThat( + rowStrideReader.getClass().getName(), + equalTo("org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader$IgnoredSourceRowStrideReader") + ); + + // assert values + assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType, true), equalTo(expected)); + } + } + } + + /** + * Returns a source only mapped field type. This is useful, since the available build() function doesn't override isParsedFromSource() + */ + private IpScriptFieldType simpleSourceOnlyMappedFieldType() { + Script script = new Script(ScriptType.INLINE, "test", "", emptyMap()); + IpFieldScript.Factory factory = new IpFieldScript.Factory() { + @Override + public IpFieldScript.LeafFactory newFactory( + String fieldName, + Map params, + SearchLookup searchLookup, + OnScriptError onScriptError + ) { + return ctx -> new IpFieldScript(fieldName, params, searchLookup, onScriptError, ctx) { + @Override + @SuppressWarnings("unchecked") + public void execute() { + Map source = (Map) this.getParams().get("_source"); + for (Object foo : (List) source.get("test")) { + try { + emit(foo.toString()); + } catch (Exception e) { + // skip + } + } + } + }; + } + + @Override + public boolean isParsedFromSource() { + return true; + } + }; + return new IpScriptFieldType("test", factory, script, emptyMap(), OnScriptError.FAIL); + } + @Override protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx) { return ft.termsQuery(randomList(100, () -> randomIp(randomBoolean())), ctx);