Skip to content

Commit 9b38ddf

Browse files
jordan-powersgmjehovich
jordan-powers
authored and
gmjehovich
committed
Patterned text conditionally disable templating (elastic#134744)
Follow-up to elastic#134466 to actually check the disable_templating parameter in the patterned_text mapper. When the parameter is set, values will be stored as-is in a stored field.
1 parent eb7ba48 commit 9b38ddf

File tree

5 files changed

+121
-25
lines changed

5 files changed

+121
-25
lines changed

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
package org.elasticsearch.xpack.logsdb.patternedtext;
99

10+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
11+
1012
import org.elasticsearch.client.Request;
1113
import org.elasticsearch.client.Response;
1214
import org.elasticsearch.common.settings.Settings;
@@ -45,6 +47,17 @@ protected String getTestRestCluster() {
4547
return cluster.getHttpAddresses();
4648
}
4749

50+
@ParametersFactory(argumentFormatting = "disableTemplating=%b")
51+
public static List<Object[]> args() {
52+
return List.of(new Object[] { true }, new Object[] { false });
53+
}
54+
55+
private final boolean disableTemplating;
56+
57+
public PatternedTextBasicRestIT(boolean disableTemplating) {
58+
this.disableTemplating = disableTemplating;
59+
}
60+
4861
@SuppressWarnings("unchecked")
4962
public void testBulkInsertThenMatchAllSource() throws IOException {
5063

@@ -63,11 +76,12 @@ public void testBulkInsertThenMatchAllSource() throws IOException {
6376
"type": "date"
6477
},
6578
"message": {
66-
"type": "patterned_text"
79+
"type": "patterned_text",
80+
"disable_templating": %disable_templating%
6781
}
6882
}
6983
}
70-
""";
84+
""".replace("%disable_templating%", Boolean.toString(disableTemplating));
7185

7286
String indexName = "test-index";
7387
createIndex(indexName, settings.build(), mapping);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,11 @@
3030
import org.elasticsearch.index.mapper.MapperBuilderContext;
3131
import org.elasticsearch.index.mapper.MapperParsingException;
3232
import org.elasticsearch.index.mapper.MappingParserContext;
33+
import org.elasticsearch.index.mapper.SourceLoader;
34+
import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
3335
import org.elasticsearch.index.mapper.TextParams;
3436
import org.elasticsearch.index.mapper.TextSearchInfo;
37+
import org.elasticsearch.xcontent.XContentBuilder;
3538

3639
import java.io.IOException;
3740
import java.util.ArrayList;
@@ -261,12 +264,17 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
261264
throw new IllegalArgumentException("Multiple values are not allowed for field [" + fieldType().name() + "].");
262265
}
263266

264-
// Parse template and args
265-
PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);
266-
267267
// Add index on original value
268268
context.doc().add(new Field(fieldType().name(), value, fieldType));
269269

270+
if (fieldType().disableTemplating()) {
271+
context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value)));
272+
return;
273+
}
274+
275+
// Parse template and args
276+
PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);
277+
270278
// Add template_id doc_values
271279
context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId())));
272280

@@ -305,14 +313,25 @@ interface DocValuesSupplier {
305313

306314
@Override
307315
protected SyntheticSourceSupport syntheticSourceSupport() {
308-
return new SyntheticSourceSupport.Native(
309-
() -> new CompositeSyntheticFieldLoader(
310-
leafName(),
311-
fullPath(),
312-
new PatternedTextSyntheticFieldLoaderLayer(
313-
fieldType().name(),
314-
leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType())
315-
)
316+
return new SyntheticSourceSupport.Native(this::getSyntheticFieldLoader);
317+
}
318+
319+
private SourceLoader.SyntheticFieldLoader getSyntheticFieldLoader() {
320+
if (fieldType().disableTemplating()) {
321+
return new StringStoredFieldFieldLoader(fieldType().storedNamed(), fieldType().name(), leafName()) {
322+
@Override
323+
protected void write(XContentBuilder b, Object value) throws IOException {
324+
b.value(((BytesRef) value).utf8ToString());
325+
}
326+
};
327+
}
328+
329+
return new CompositeSyntheticFieldLoader(
330+
leafName(),
331+
fullPath(),
332+
new PatternedTextSyntheticFieldLoaderLayer(
333+
fieldType().name(),
334+
leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType())
316335
)
317336
);
318337
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@
2929
import org.elasticsearch.index.fielddata.FieldDataContext;
3030
import org.elasticsearch.index.fielddata.IndexFieldData;
3131
import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData;
32+
import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
3233
import org.elasticsearch.index.mapper.BlockLoader;
34+
import org.elasticsearch.index.mapper.BlockStoredFieldsReader;
3335
import org.elasticsearch.index.mapper.SourceValueFetcher;
3436
import org.elasticsearch.index.mapper.StringFieldType;
3537
import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -49,6 +51,7 @@
4951
import java.util.List;
5052
import java.util.Map;
5153
import java.util.Objects;
54+
import java.util.Set;
5255

5356
public class PatternedTextFieldType extends StringFieldType {
5457

@@ -118,6 +121,10 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
118121
private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> getValueFetcherProvider(
119122
SearchExecutionContext searchExecutionContext
120123
) {
124+
if (disableTemplating) {
125+
return storedFieldFetcher(storedNamed());
126+
}
127+
121128
return context -> {
122129
ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null);
123130
SourceProvider sourceProvider = searchExecutionContext.lookup();
@@ -132,6 +139,18 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
132139
};
133140
}
134141

142+
private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> storedFieldFetcher(String name) {
143+
var loader = StoredFieldLoader.create(false, Set.of(name));
144+
return context -> {
145+
var leafLoader = loader.getLoader(context, null);
146+
return docId -> {
147+
leafLoader.advanceTo(docId);
148+
var storedFields = leafLoader.storedFields();
149+
return storedFields.get(name);
150+
};
151+
};
152+
}
153+
135154
private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext context) {
136155
// Disable scoring similarly to match_only_text
137156
if (hasPositions) {
@@ -262,6 +281,10 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions,
262281

263282
@Override
264283
public BlockLoader blockLoader(BlockLoaderContext blContext) {
284+
if (disableTemplating) {
285+
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedNamed());
286+
}
287+
265288
return new PatternedTextBlockLoader((leafReader -> PatternedTextCompositeValues.from(leafReader, this)));
266289
}
267290

‎x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,16 @@ public void testPhraseQuerySyntheticSource() throws IOException {
9797
assertPhraseQuery(createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text"))));
9898
}
9999

100+
public void testPhraseQueryStandardSourceDisableTemplating() throws IOException {
101+
assertPhraseQuery(createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true))));
102+
}
103+
104+
public void testPhraseQuerySyntheticSourceDisableTemplating() throws IOException {
105+
assertPhraseQuery(
106+
createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true)))
107+
);
108+
}
109+
100110
private void assertPhraseQuery(MapperService mapperService) throws IOException {
101111
try (Directory directory = newDirectory()) {
102112
RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
@@ -322,6 +332,9 @@ private Tuple<String, String> generateValue() {
322332

323333
private void mapping(XContentBuilder b) throws IOException {
324334
b.field("type", "patterned_text");
335+
if (randomBoolean()) {
336+
b.field("disable_templating", true);
337+
}
325338
}
326339

327340
@Override

‎x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
package org.elasticsearch.xpack.logsdb.patternedtext;
99

10+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
11+
1012
import org.apache.logging.log4j.LogManager;
1113
import org.apache.logging.log4j.Logger;
1214
import org.elasticsearch.action.DocWriteRequest;
@@ -41,6 +43,7 @@
4143
import java.util.ArrayList;
4244
import java.util.Arrays;
4345
import java.util.Collection;
46+
import java.util.Collections;
4447
import java.util.HashSet;
4548
import java.util.List;
4649
import java.util.Map;
@@ -56,6 +59,27 @@
5659
public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
5760
private static final Logger logger = LogManager.getLogger(PatternedTextIntegrationTests.class);
5861

62+
@ParametersFactory(argumentFormatting = "indexOptions=%s, disableTemplating=%b")
63+
public static List<Object[]> args() {
64+
List<Object[]> args = new ArrayList<>();
65+
for (var indexOption : new String[] { "docs", "positions" }) {
66+
for (var templating : new boolean[] { true, false }) {
67+
args.add(new Object[] { indexOption, templating });
68+
}
69+
}
70+
return Collections.unmodifiableList(args);
71+
}
72+
73+
private final String indexOptions;
74+
private final boolean disableTemplating;
75+
private final String mapping;
76+
77+
public PatternedTextIntegrationTests(String indexOptions, boolean disableTemplating) {
78+
this.indexOptions = indexOptions;
79+
this.disableTemplating = disableTemplating;
80+
this.mapping = getMapping(indexOptions, disableTemplating);
81+
}
82+
5983
@Override
6084
protected Settings nodeSettings() {
6185
return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build();
@@ -75,17 +99,15 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
7599
"@timestamp": { "type": "date" },
76100
"field_match_only_text": { "type": "match_only_text" },
77101
"field_patterned_text": {
78-
"type": "patterned_text",
79-
"index_options": "%",
80-
"analyzer": "standard"
102+
"type": "patterned_text",
103+
"index_options": "%index_options%",
104+
"disable_templating": "%disable_templating%",
105+
"analyzer": "standard"
81106
}
82107
}
83108
}
84109
""";
85110

86-
private static final String MAPPING_DOCS_ONLY = MAPPING_TEMPLATE.replace("%", "docs");
87-
private static final String MAPPING_POSITIONS = MAPPING_TEMPLATE.replace("%", "positions");
88-
89111
private static final Settings LOGSDB_SETTING = Settings.builder().put(IndexSettings.MODE.getKey(), "logsdb").build();
90112

91113
@Before
@@ -100,8 +122,12 @@ public void cleanup() {
100122
}
101123
}
102124

125+
private String getMapping(String indexOptions, boolean disableTemplating) {
126+
return MAPPING_TEMPLATE.replace("%index_options%", indexOptions)
127+
.replace("%disable_templating%", Boolean.toString(disableTemplating));
128+
}
129+
103130
public void testSourceMatchAllManyValues() throws IOException {
104-
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
105131
var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
106132
createIndex(INDEX, createRequest);
107133

@@ -114,7 +140,6 @@ public void testSourceMatchAllManyValues() throws IOException {
114140
}
115141

116142
public void testLargeValueIsStored() throws IOException {
117-
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
118143
var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
119144
IndexService indexService = createIndex(INDEX, createRequest);
120145

@@ -136,7 +161,6 @@ public void testLargeValueIsStored() throws IOException {
136161
}
137162

138163
public void testSmallValueNotStored() throws IOException {
139-
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
140164
var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
141165
IndexService indexService = createIndex(INDEX, createRequest);
142166

@@ -148,17 +172,20 @@ public void testSmallValueNotStored() throws IOException {
148172
assertMappings();
149173
assertMessagesInSource(messages);
150174

151-
// assert does not contain stored field
175+
// assert only contains stored field if templating is disabled
152176
try (var searcher = indexService.getShard(0).acquireSearcher(INDEX)) {
153177
try (var indexReader = searcher.getIndexReader()) {
154178
var document = indexReader.storedFields().document(0);
155-
assertNull(document.getField("field_patterned_text.stored"));
179+
if (disableTemplating) {
180+
assertEquals(document.getField("field_patterned_text.stored").binaryValue().utf8ToString(), message);
181+
} else {
182+
assertNull(document.getField("field_patterned_text.stored"));
183+
}
156184
}
157185
}
158186
}
159187

160188
public void testQueryResultsSameAsMatchOnlyText() throws IOException {
161-
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
162189
var createRequest = new CreateIndexRequest(INDEX).mapping(mapping);
163190

164191
if (randomBoolean()) {

0 commit comments

Comments
 (0)