Skip to content

Commit f11085e

Browse files
parkertimminsparkertimmins
committed
Working, but still quite messy
1 parent 3f348aa commit f11085e

File tree

1 file changed

+217
-25
lines changed

1 file changed

+217
-25
lines changed

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java‎

Lines changed: 217 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,26 @@
1717
import org.elasticsearch.common.time.DateFormatter;
1818
import org.elasticsearch.common.time.FormatNames;
1919
import org.elasticsearch.common.xcontent.XContentHelper;
20+
import org.elasticsearch.datageneration.FieldType;
21+
import org.elasticsearch.datageneration.Template;
22+
import org.elasticsearch.datageneration.fields.leaf.BooleanFieldDataGenerator;
23+
import org.elasticsearch.datageneration.fields.leaf.ByteFieldDataGenerator;
24+
import org.elasticsearch.datageneration.fields.leaf.ConstantKeywordFieldDataGenerator;
25+
import org.elasticsearch.datageneration.fields.leaf.CountedKeywordFieldDataGenerator;
26+
import org.elasticsearch.datageneration.fields.leaf.DateFieldDataGenerator;
27+
import org.elasticsearch.datageneration.fields.leaf.DoubleFieldDataGenerator;
28+
import org.elasticsearch.datageneration.fields.leaf.FloatFieldDataGenerator;
29+
import org.elasticsearch.datageneration.fields.leaf.GeoPointFieldDataGenerator;
30+
import org.elasticsearch.datageneration.fields.leaf.HalfFloatFieldDataGenerator;
31+
import org.elasticsearch.datageneration.fields.leaf.IntegerFieldDataGenerator;
32+
import org.elasticsearch.datageneration.fields.leaf.IpFieldDataGenerator;
33+
import org.elasticsearch.datageneration.fields.leaf.KeywordFieldDataGenerator;
34+
import org.elasticsearch.datageneration.fields.leaf.LongFieldDataGenerator;
35+
import org.elasticsearch.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
36+
import org.elasticsearch.datageneration.fields.leaf.ShortFieldDataGenerator;
37+
import org.elasticsearch.datageneration.fields.leaf.TextFieldDataGenerator;
38+
import org.elasticsearch.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
39+
import org.elasticsearch.datageneration.fields.leaf.WildcardFieldDataGenerator;
2040
import org.elasticsearch.datageneration.matchers.MatchResult;
2141
import org.elasticsearch.datageneration.matchers.Matcher;
2242
import org.elasticsearch.datageneration.matchers.source.SourceTransforms;
@@ -44,7 +64,9 @@
4464
import java.util.Comparator;
4565
import java.util.List;
4666
import java.util.Map;
67+
import java.util.Objects;
4768
import java.util.TreeMap;
69+
import java.util.stream.Collectors;
4870

4971
import static org.hamcrest.Matchers.equalTo;
5072
import static org.hamcrest.Matchers.greaterThan;
@@ -145,62 +167,92 @@ public void testMatchAllQuery() throws IOException {
145167
assertTrue(matchResult.getMessage(), matchResult.isMatch());
146168
}
147169

148-
private List<String> getFieldsOfType(String type, Map<String, Map<String, Object>> mappingLookup) {
149-
return mappingLookup.entrySet().stream()
150-
.filter(e -> {
151-
var mapping = e.getValue();
152-
return mapping != null && type.equals(mapping.get("type"));
153-
})
154-
.map(Map.Entry::getKey)
155-
.toList();
156-
}
157-
158170
private List<String> getSearchableFields(String type, Map<String, Map<String, Object>> mappingLookup) {
159171
var fields = new ArrayList<String>();
160172
for (var e : mappingLookup.entrySet()) {
161173
var mapping = e.getValue();
162174
if (mapping != null && type.equals(mapping.get("type"))) {
163-
boolean isIndexed = "false".equals(mapping.get("index")) == false;
164-
boolean hasDocValues = "false".equals(mapping.get("doc_values")) == false;
165-
if (isIndexed || hasDocValues) {
175+
boolean isIndexed = (Boolean) mapping.getOrDefault("index", true);
176+
if (isIndexed) {
166177
fields.add(e.getKey());
167178
}
168179
}
169180
}
170181
return fields;
171182
}
172183

173-
private QueryBuilder nestedPhraseQuery(String path, String phrase) {
184+
@SuppressWarnings("unchecked")
185+
List<String> getNestedPathPrefixes(String[] path) {
186+
Map<String, Object> mapping = dataGenerationHelper.mapping().raw();
187+
mapping = (Map<String, Object>) mapping.get("_doc");
188+
mapping = (Map<String, Object>) mapping.get("properties");
189+
190+
var result = new ArrayList<String>();
191+
for (int i = 0; i < path.length - 1; i++) {
192+
var field = path[i];
193+
mapping = (Map<String, Object>) mapping.get(field);
194+
boolean nested = "nested".equals(mapping.get("type"));
195+
if (nested) {
196+
result.add(String.join(".", Arrays.copyOfRange(path, 0, i + 1)));
197+
}
198+
mapping = (Map<String, Object>) mapping.get("properties");
199+
}
200+
201+
mapping = (Map<String, Object>) mapping.get(path[path.length - 1]);
202+
assert mapping.containsKey("properties") == false;
203+
return result;
204+
}
205+
206+
private QueryBuilder wrapInNestedQuery(String path, QueryBuilder leafQuery) {
207+
String[] parts = path.split("\\.");
208+
List<String> nestedPaths = getNestedPathPrefixes(parts);
209+
QueryBuilder query = leafQuery;
210+
for (String nestedPath : nestedPaths.reversed()) {
211+
query = QueryBuilders.nestedQuery(nestedPath, query, ScoreMode.Max);
212+
}
213+
return query;
214+
}
215+
216+
private boolean isEnabled(String path) {
174217
String[] parts = path.split("\\.");
175-
List<Boolean> nested = dataGenerationHelper.isNested(parts);
218+
var mappingLookup = dataGenerationHelper.mapping().lookup();
219+
for (int i = 0; i < parts.length - 1; i++) {
220+
var pathToHere = String.join(".", Arrays.copyOfRange(parts, 0, i + 1));
221+
Map<String, Object> mapping = mappingLookup.get(pathToHere);
222+
176223

177-
QueryBuilder query = QueryBuilders.matchPhraseQuery(path, phrase);
178-
for (int i = parts.length - 2; i >= 0; i--) {
179-
if (nested.get(i)) {
180-
var pathToObject = String.join(".", Arrays.copyOfRange(parts, 0, i + 1));
181-
query = QueryBuilders.nestedQuery(pathToObject, query, ScoreMode.Max);
224+
boolean enabled = true;
225+
if (mapping.containsKey("enabled") && mapping.get("enabled") instanceof Boolean) {
226+
enabled = (Boolean) mapping.get("enabled");
227+
}
228+
if (mapping.containsKey("enabled") && mapping.get("enabled") instanceof String) {
229+
enabled = Boolean.parseBoolean((String) mapping.get("enabled"));
230+
}
231+
232+
if (enabled == false) {
233+
return false;
182234
}
183235
}
184-
return query;
236+
return true;
185237
}
186238

187239
public void testPhraseQuery() throws IOException {
188240
int numberOfDocuments = ESTestCase.randomIntBetween(20, 80);
189241
final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
190242

191243
var mappingLookup = dataGenerationHelper.mapping().lookup();
192-
var fieldsOfType = getSearchableFields("keyword", mappingLookup);
244+
var fieldsOfType = getSearchableFields("text", mappingLookup);
193245

194246
if (fieldsOfType.isEmpty()) {
195247
return;
196248
}
197249

198-
var field = randomFrom(fieldsOfType);
250+
var path = randomFrom(fieldsOfType);
199251

200252
XContentBuilder doc = randomFrom(documents);
201253
final Map<String, Object> document = XContentHelper.convertToMap(XContentType.JSON.xContent(), Strings.toString(doc), true);
202254
var normalized = SourceTransforms.normalize(document, mappingLookup);
203-
List<Object> values = normalized.get(field);
255+
List<Object> values = normalized.get(path);
204256
if (values == null || values.isEmpty()) {
205257
return;
206258
}
@@ -218,7 +270,7 @@ public void testPhraseQuery() throws IOException {
218270

219271
indexDocuments(documents);
220272

221-
QueryBuilder queryBuilder = nestedPhraseQuery(field, phrase);
273+
QueryBuilder queryBuilder = wrapInNestedQuery(path, QueryBuilders.matchPhraseQuery(path, phrase));
222274
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder)
223275
.size(numberOfDocuments);
224276

@@ -231,6 +283,134 @@ public void testPhraseQuery() throws IOException {
231283
assertTrue(matchResult.getMessage(), matchResult.isMatch());
232284
}
233285

286+
private static QueryBuilder phraseQuery(String path, Object value) {
287+
String needle = (String) value;
288+
var tokens = Arrays.asList(needle.split("[^a-zA-Z0-9]"));
289+
290+
if (tokens.isEmpty()) {
291+
return null;
292+
}
293+
294+
int low = ESTestCase.randomIntBetween(0, tokens.size() - 1);
295+
int hi = ESTestCase.randomIntBetween(low+1, tokens.size());
296+
var phrase = String.join(" ", tokens.subList(low, hi));
297+
298+
return QueryBuilders.matchPhraseQuery(path, phrase);
299+
}
300+
301+
private static List<QueryBuilder> buildLeafQueries(String type, String path, Object value, Map<String, Object> mapping) {
302+
FieldType fieldType = FieldType.tryParse(type);
303+
if (fieldType == null) {
304+
return List.of();
305+
}
306+
return switch (fieldType) {
307+
case KEYWORD -> {
308+
var ignoreAbove = (Integer) mapping.getOrDefault("ignore_above", Integer.MAX_VALUE);
309+
yield ignoreAbove >= ((String) value).length() ? List.of(QueryBuilders.termQuery(path, value)) : List.of();
310+
}
311+
case TEXT -> {
312+
List<QueryBuilder> result = new ArrayList<>(List.of(QueryBuilders.matchQuery(path, value)));
313+
var phrase = phraseQuery(path, value);
314+
if (phrase != null) {
315+
result.add(phrase);
316+
}
317+
yield result;
318+
}
319+
case WILDCARD -> {
320+
var ignoreAbove = (Integer) mapping.getOrDefault("ignore_above", Integer.MAX_VALUE);
321+
if (ignoreAbove >= ((String) value).length()) {
322+
yield List.of(
323+
QueryBuilders.termQuery(path, value),
324+
QueryBuilders.wildcardQuery(path, value + "*")
325+
);
326+
}
327+
yield List.of();
328+
}
329+
default -> List.of();
330+
};
331+
}
332+
333+
public void testRandomQueries() throws IOException {
334+
int numberOfDocuments = ESTestCase.randomIntBetween(20, 80);
335+
final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
336+
var mappingLookup = dataGenerationHelper.mapping().lookup();
337+
final List<Map<String, List<Object>>> docsNormalized = documents.stream().map(d -> {
338+
var document = XContentHelper.convertToMap(XContentType.JSON.xContent(), Strings.toString(d), true);
339+
return SourceTransforms.normalize(document, mappingLookup);
340+
}).toList();
341+
342+
indexDocuments(documents);
343+
344+
for (var e : mappingLookup.entrySet()) {
345+
var path = e.getKey();
346+
var mapping = e.getValue();
347+
348+
// This test cannot handle fields with periods in name
349+
if (path.equals("host.name")) {
350+
continue;
351+
}
352+
if (mapping == null || isEnabled(path) == false) {
353+
continue;
354+
}
355+
boolean isIndexed = (Boolean) mapping.getOrDefault("index", true);
356+
if (isIndexed == false) {
357+
continue;
358+
}
359+
var docsWithFields = docsNormalized.stream().filter(d -> d.containsKey(path)).toList();
360+
if (docsWithFields.isEmpty()) {
361+
continue;
362+
}
363+
364+
var doc = randomFrom(docsWithFields);
365+
List<Object> values = doc.get(path);
366+
if (values == null) {
367+
continue;
368+
}
369+
List<Object> valuesNonNull = values.stream().filter(Objects::nonNull).toList();
370+
if (valuesNonNull.isEmpty()) {
371+
continue;
372+
}
373+
374+
Object needle = randomFrom(valuesNonNull);
375+
376+
var type = (String) mapping.get("type");
377+
var leafQueries = buildLeafQueries(type, path, needle, mapping).stream().filter(Objects::nonNull).toList();
378+
379+
for (var leafQuery : leafQueries) {
380+
QueryBuilder queryBuilder = wrapInNestedQuery(path, leafQuery);
381+
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder)
382+
.size(numberOfDocuments);
383+
384+
logger.info("Querying for field [{}] with value [{}]", path, needle);
385+
final MatchResult matchResult = Matcher.matchSource()
386+
.mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings())
387+
.settings(getContenderSettings(), getBaselineSettings())
388+
.expected(getQueryHits(queryBaseline(searchSourceBuilder)))
389+
.ignoringSort(true)
390+
.isEqualTo(getQueryHits(queryContender(searchSourceBuilder)));
391+
assertTrue(matchResult.getMessage(), matchResult.isMatch());
392+
}
393+
}
394+
}
395+
396+
public void testTermsQuery() throws IOException {
397+
int numberOfDocuments = ESTestCase.randomIntBetween(20, 80);
398+
final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
399+
400+
indexDocuments(documents);
401+
402+
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.termQuery("method", "put"))
403+
.size(numberOfDocuments);
404+
405+
final MatchResult matchResult = Matcher.matchSource()
406+
.mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings())
407+
.settings(getContenderSettings(), getBaselineSettings())
408+
.expected(getQueryHits(queryBaseline(searchSourceBuilder)))
409+
.ignoringSort(true)
410+
.isEqualTo(getQueryHits(queryContender(searchSourceBuilder)));
411+
assertTrue(matchResult.getMessage(), matchResult.isMatch());
412+
}
413+
234414
public void testHistogramAggregation() throws IOException {
235415
int numberOfDocuments = ESTestCase.randomIntBetween(20, 80);
236416
final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
@@ -365,6 +545,18 @@ protected XContentBuilder generateDocument(final Instant timestamp) throws IOExc
365545
return document;
366546
}
367547

548+
@SuppressWarnings("unchecked")
549+
private List<Map<String, Object>> hits(QueryBuilder query) throws IOException {
550+
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(query)
551+
.size(10);
552+
553+
Response response = queryBaseline(searchSourceBuilder);
554+
555+
final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);
556+
final Map<String, Object> hitsMap = (Map<String, Object>) map.get("hits");
557+
return (List<Map<String, Object>>) hitsMap.get("hits");
558+
}
559+
368560
@SuppressWarnings("unchecked")
369561
private static List<Map<String, Object>> getQueryHits(final Response response) throws IOException {
370562
final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);

0 commit comments

Comments
 (0)