From b57749754fc53831e9bf10ed2781e85046a45d59 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 24 Sep 2025 15:07:20 -0700 Subject: [PATCH 1/3] Improve block loader for source only runtime IP fields --- .../index/mapper/IpScriptFieldType.java | 46 ++++++ .../index/mapper/IpScriptFieldTypeTests.java | 143 ++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index 9923235f09025..a7c0322868414 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -33,7 +33,9 @@ import org.elasticsearch.search.runtime.IpScriptFieldRangeQuery; import org.elasticsearch.search.runtime.IpScriptFieldTermQuery; import org.elasticsearch.search.runtime.IpScriptFieldTermsQuery; +import org.elasticsearch.xcontent.XContentParser; +import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; import java.util.ArrayList; @@ -213,6 +215,50 @@ private Query cidrQuery(String term, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { + FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader = fallbackSyntheticSourceBlockLoader( + blContext, + BlockLoader.BlockFactory::bytesRefs, + this::fallbackSyntheticSourceBlockLoaderReader + ); + + if (fallbackSyntheticSourceBlockLoader != null) { + return fallbackSyntheticSourceBlockLoader; + } return new IpScriptBlockDocValuesReader.IpScriptBlockLoader(leafFactory(blContext.lookup())); } + + private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(null) { + @Override + public void convertValue(Object value, List accumulator) { + try { + if (value instanceof InetAddress ia) { + accumulator.add(ia); + } else { + accumulator.add(InetAddresses.forString(value.toString())); + } + } catch (Exception e) { + // value is malformed, skip it + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + BlockLoader.BytesRefBuilder builder = (BlockLoader.BytesRefBuilder) blockBuilder; + for (InetAddress addr : values) { + builder.appendBytesRef(new BytesRef(InetAddressPoint.encode(addr))); + } + } + + @Override + protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + try { + accumulator.add(InetAddresses.forString(parser.text())); + } catch (Exception e) { + // value is malformed, skip it + } + } + }; + } + } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index 7e9a236f6cc74..81e85d7447f7b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -30,6 +30,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.BinaryScriptFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues.Strings; @@ -43,20 +44,27 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; public class IpScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase { + private static final BytesRef EMPTY_IP = null; + private static final BytesRef MALFORMED_IP = null; + @Override protected ScriptFactory parseFromSource() { return IpFieldScript.PARSE_FROM_SOURCE; @@ -280,6 +288,141 @@ public void testBlockLoader() throws IOException { } } + public void testBlockLoaderSourceOnlyRuntimeField() throws IOException { + try ( + Directory directory = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE)) + ) { + // given + // try multiple variations of boolean as they're all encoded slightly differently + iw.addDocuments( + List.of( + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"192.168.0.1\"]}"))), + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"2001:db8::1\"]}"))), + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"\"]}"))), + // ensure a malformed value doesn't crash + List.of(new StoredField("_source", new BytesRef("{\"test\": [\"potato\"]}"))) + ) + ); + IpScriptFieldType fieldType = simpleSourceOnlyMappedFieldType(); + List expected = Arrays.asList( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8::1"))), + EMPTY_IP, + MALFORMED_IP + ); + + try (DirectoryReader reader = iw.getReader()) { + // when + BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY, true)); + + // then + + // assert loader is of expected instance type + assertThat(loader, instanceOf(IpScriptBlockDocValuesReader.IpScriptBlockLoader.class)); + + // ignored source doesn't support column at a time loading: + var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst()); + assertThat(columnAtATimeLoader, instanceOf(IpScriptBlockDocValuesReader.class)); + + var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst()); + assertThat(rowStrideReader, instanceOf(IpScriptBlockDocValuesReader.class)); + + // assert values + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 0), equalTo(expected)); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected)); + } + } + } + + public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IOException { + try ( + Directory directory = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE)) + ) { + // given + // try multiple variations of boolean as they're all encoded slightly differently + iw.addDocuments( + List.of( + createDocumentWithIgnoredSource("[\"192.168.0.1\"]"), + createDocumentWithIgnoredSource("[\"2001:db8::1\"]"), + createDocumentWithIgnoredSource("[\"\"]"), + // ensure a malformed value doesn't crash + createDocumentWithIgnoredSource("[\"potato\"]") + ) + ); + + Settings settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build(); + IpScriptFieldType fieldType = simpleSourceOnlyMappedFieldType(); + List expected = Arrays.asList( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8::1"))), + EMPTY_IP, + MALFORMED_IP + ); + + try (DirectoryReader reader = iw.getReader()) { + // when + BlockLoader loader = fieldType.blockLoader(blContext(settings, true)); + + // then + + // assert loader is of expected instance type + assertThat(loader, instanceOf(FallbackSyntheticSourceBlockLoader.class)); + + // ignored source doesn't support column at a time loading: + var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst()); + assertThat(columnAtATimeLoader, nullValue()); + + var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst()); + assertThat( + rowStrideReader.getClass().getName(), + equalTo("org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader$IgnoredSourceRowStrideReader") + ); + + // assert values + assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType, true), equalTo(expected)); + } + } + } + + /** + * Returns a source only mapped field type. This is useful, since the available build() function doesn't override isParsedFromSource() + */ + private IpScriptFieldType simpleSourceOnlyMappedFieldType() { + Script script = new Script(ScriptType.INLINE, "test", "", emptyMap()); + IpFieldScript.Factory factory = new IpFieldScript.Factory() { + @Override + public IpFieldScript.LeafFactory newFactory( + String fieldName, + Map params, + SearchLookup searchLookup, + OnScriptError onScriptError + ) { + return ctx -> new IpFieldScript(fieldName, params, searchLookup, onScriptError, ctx) { + @Override + @SuppressWarnings("unchecked") + public void execute() { + Map source = (Map) this.getParams().get("_source"); + for (Object foo : (List) source.get("test")) { + try { + emit(foo.toString()); + } catch (Exception e) { + // skip + } + } + } + }; + } + + @Override + public boolean isParsedFromSource() { + return true; + } + }; + return new IpScriptFieldType("test", factory, script, emptyMap(), OnScriptError.FAIL); + } + @Override protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx) { return ft.termsQuery(randomList(100, () -> randomIp(randomBoolean())), ctx); From ddd96ab8897d16ce1dcdbaf5858086e39558e697 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Thu, 25 Sep 2025 12:46:59 -0700 Subject: [PATCH 2/3] Update docs/changelog/135393.yaml --- docs/changelog/135393.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135393.yaml diff --git a/docs/changelog/135393.yaml b/docs/changelog/135393.yaml new file mode 100644 index 0000000000000..c80e101355878 --- /dev/null +++ b/docs/changelog/135393.yaml @@ -0,0 +1,5 @@ +pr: 135393 +summary: Improve block loader for source only runtime IP fields +area: Mapping +type: enhancement +issues: [] From 958045974a866ad4f415ba0489f84dc42d412272 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Thu, 25 Sep 2025 12:58:18 -0700 Subject: [PATCH 3/3] Extracted ip block reader into a separate class --- .../IpFallbackSyntheticSourceReader.java | 60 +++++++++++++++++++ .../index/mapper/IpFieldMapper.java | 41 +------------ .../index/mapper/IpScriptFieldType.java | 38 +----------- 3 files changed, 62 insertions(+), 77 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java new file mode 100644 index 0000000000000..1d837ea9daddb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + +public class IpFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader { + + public IpFallbackSyntheticSourceReader(Object nullValue) { + super(nullValue); + } + + @Override + public void convertValue(Object value, List accumulator) { + try { + if (value instanceof InetAddress ia) { + accumulator.add(ia); + } else { + InetAddress address = InetAddresses.forString(value.toString()); + accumulator.add(address); + } + } catch (Exception e) { + // value is malformed, skip it + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + BlockLoader.BytesRefBuilder builder = (BlockLoader.BytesRefBuilder) blockBuilder; + for (InetAddress address : values) { + var bytesRef = new BytesRef(InetAddressPoint.encode(address)); + builder.appendBytesRef(bytesRef); + } + } + + @Override + protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + try { + InetAddress address = InetAddresses.forString(parser.text()); + accumulator.add(address); + } catch (Exception e) { + // value is malformed, skip it + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 3781242815e95..781e4e27752a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -42,7 +42,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentString; import java.io.IOException; @@ -52,7 +51,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -480,44 +478,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { - var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { - @Override - public void convertValue(Object value, List accumulator) { - if (value instanceof InetAddress ia) { - accumulator.add(ia); - } - - try { - var address = InetAddresses.forString(value.toString()); - accumulator.add(address); - } catch (Exception e) { - // Malformed value, skip it - } - } - - @Override - protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { - // aligned with #parseCreateField() - String value = parser.text(); - - try { - var address = InetAddresses.forString(value); - accumulator.add(address); - } catch (Exception e) { - // Malformed value, skip it - } - } - - @Override - public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { - var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder; - - for (var value : values) { - bytesRefBuilder.appendBytesRef(new BytesRef(InetAddressPoint.encode(value))); - } - } - }; - + var reader = new IpFallbackSyntheticSourceReader(nullValue); return new FallbackSyntheticSourceBlockLoader( reader, name(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index a7c0322868414..8c8c3f004dcb1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -33,9 +33,7 @@ import org.elasticsearch.search.runtime.IpScriptFieldRangeQuery; import org.elasticsearch.search.runtime.IpScriptFieldTermQuery; import org.elasticsearch.search.runtime.IpScriptFieldTermsQuery; -import org.elasticsearch.xcontent.XContentParser; -import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; import java.util.ArrayList; @@ -218,7 +216,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader = fallbackSyntheticSourceBlockLoader( blContext, BlockLoader.BlockFactory::bytesRefs, - this::fallbackSyntheticSourceBlockLoaderReader + () -> new IpFallbackSyntheticSourceReader(null) ); if (fallbackSyntheticSourceBlockLoader != null) { @@ -227,38 +225,4 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { return new IpScriptBlockDocValuesReader.IpScriptBlockLoader(leafFactory(blContext.lookup())); } - private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { - return new FallbackSyntheticSourceBlockLoader.SingleValueReader(null) { - @Override - public void convertValue(Object value, List accumulator) { - try { - if (value instanceof InetAddress ia) { - accumulator.add(ia); - } else { - accumulator.add(InetAddresses.forString(value.toString())); - } - } catch (Exception e) { - // value is malformed, skip it - } - } - - @Override - public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { - BlockLoader.BytesRefBuilder builder = (BlockLoader.BytesRefBuilder) blockBuilder; - for (InetAddress addr : values) { - builder.appendBytesRef(new BytesRef(InetAddressPoint.encode(addr))); - } - } - - @Override - protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { - try { - accumulator.add(InetAddresses.forString(parser.text())); - } catch (Exception e) { - // value is malformed, skip it - } - } - }; - } - }