diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceTransforms.java b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceTransforms.java index 6a8c77b27fe8b..92b44eff0ddbf 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceTransforms.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceTransforms.java @@ -18,7 +18,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -class SourceTransforms { +public class SourceTransforms { /** * This preprocessing step makes it easier to match the document using a unified structure. * It performs following modifications: diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java new file mode 100644 index 0000000000000..be26db580edf6 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datageneration.queries; + +import org.elasticsearch.datageneration.FieldType; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public interface LeafQueryGenerator { + + List generate(Map fieldMapping, String path, Object value); + + /** + * Build a query for a specific type. If the field is nested, this query will need to be wrapped in nested queries. + * @param type the type to build a query for + * @return a generator that can build queries for this type + */ + static LeafQueryGenerator buildForType(String type) { + LeafQueryGenerator noQueries = (Map fieldMapping, String path, Object value) -> List.of(); + + FieldType fieldType = FieldType.tryParse(type); + if (fieldType == null) { + return noQueries; + } + + return switch (fieldType) { + case KEYWORD -> new KeywordQueryGenerator(); + case TEXT -> new TextQueryGenerator(); + case WILDCARD -> new WildcardQueryGenerator(); + default -> noQueries; + }; + } + + class KeywordQueryGenerator implements LeafQueryGenerator { + public List generate(Map fieldMapping, String path, Object value) { + if (fieldMapping != null) { + boolean isIndexed = (Boolean) fieldMapping.getOrDefault("index", true); + boolean hasDocValues = (Boolean) fieldMapping.getOrDefault("doc_values", true); + if (isIndexed == false && hasDocValues == false) { + return List.of(); + } + } + return List.of(QueryBuilders.termQuery(path, value)); + } + } + + class WildcardQueryGenerator implements LeafQueryGenerator { + public List generate(Map fieldMapping, String path, Object value) { + // Queries with emojis can currently fail due to https://github.com/elastic/elasticsearch/issues/132144 + if (containsHighSurrogates((String) value)) { + return List.of(); + } + return List.of(QueryBuilders.termQuery(path, value), QueryBuilders.wildcardQuery(path, value + "*")); + } + } + + class TextQueryGenerator implements LeafQueryGenerator { + public List generate(Map fieldMapping, String path, Object value) { + if (fieldMapping != null) { + boolean isIndexed = (Boolean) fieldMapping.getOrDefault("index", true); + if (isIndexed == false) { + return List.of(); + } + } + + var results = new ArrayList(); + results.add(QueryBuilders.matchQuery(path, value)); + var phraseQuery = buildPhraseQuery(path, (String) value); + if (phraseQuery != null) { + results.add(phraseQuery); + } + return results; + } + + private static QueryBuilder buildPhraseQuery(String path, String value) { + var tokens = Arrays.asList(value.split("[^a-zA-Z0-9]")); + if (tokens.isEmpty()) { + return null; + } + + int low = ESTestCase.randomIntBetween(0, tokens.size() - 1); + int hi = ESTestCase.randomIntBetween(low + 1, tokens.size()); + var phrase = String.join(" ", tokens.subList(low, hi)); + return QueryBuilders.matchPhraseQuery(path, phrase); + } + } + + static boolean containsHighSurrogates(String s) { + for (int i = 0; i < s.length(); i++) { + if (Character.isHighSurrogate(s.charAt(i))) { + return true; + } + } + return false; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java new file mode 100644 index 0000000000000..9db0b628f85da --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datageneration.queries; + +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.datageneration.Mapping; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class QueryGenerator { + + private final Mapping mapping; + + public QueryGenerator(Mapping mapping) { + this.mapping = mapping; + } + + public List generateQueries(String type, String path, Object value) { + // This query generator cannot handle fields with periods in the name. + if (path.equals("host.name")) { + return List.of(); + } + // Can handle dynamically mapped fields, but not runtime fields + if (isRuntimeField(path)) { + return List.of(); + } + var leafQueryGenerator = LeafQueryGenerator.buildForType(type); + var fieldMapping = mapping.lookup().get(path); + var leafQueries = leafQueryGenerator.generate(fieldMapping, path, value); + return leafQueries.stream().map(q -> wrapInNestedQuery(path, q)).toList(); + } + + private QueryBuilder wrapInNestedQuery(String path, QueryBuilder leafQuery) { + String[] parts = path.split("\\."); + List nestedPaths = getNestedPathPrefixes(parts); + QueryBuilder query = leafQuery; + for (String nestedPath : nestedPaths.reversed()) { + query = QueryBuilders.nestedQuery(nestedPath, query, ScoreMode.Max); + } + return query; + } + + @SuppressWarnings("unchecked") + private List getNestedPathPrefixes(String[] path) { + Map mapping = this.mapping.raw(); + mapping = (Map) mapping.get("_doc"); + mapping = (Map) mapping.get("properties"); + + var result = new ArrayList(); + for (int i = 0; i < path.length - 1; i++) { + var field = path[i]; + mapping = (Map) mapping.get(field); + + // dynamic field + if (mapping == null) { + break; + } + + boolean nested = "nested".equals(mapping.get("type")); + if (nested) { + result.add(String.join(".", Arrays.copyOfRange(path, 0, i + 1))); + } + mapping = (Map) mapping.get("properties"); + } + return result; + } + + @SuppressWarnings("unchecked") + private boolean isRuntimeField(String path) { + String[] parts = path.split("\\."); + var topLevelMapping = (Map) mapping.raw().get("_doc"); + boolean inRuntimeContext = "runtime".equals(topLevelMapping.get("dynamic")); + for (int i = 0; i < parts.length - 1; i++) { + var pathToHere = String.join(".", Arrays.copyOfRange(parts, 0, i + 1)); + Map fieldMapping = mapping.lookup().get(pathToHere); + if (fieldMapping == null) { + break; + } + if (fieldMapping.containsKey("dynamic")) { + // lower down dynamic definitions override higher up behavior + inRuntimeContext = "runtime".equals(fieldMapping.get("dynamic")); + } + } + return inRuntimeContext; + } +} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java index 01d5fd35320e4..86c435ba2a4e8 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -121,6 +122,26 @@ Mapping mapping() { return this.mapping; } + public Map getTemplateFieldTypes() { + Map allPaths = new TreeMap<>(); + gatherFieldTypes(allPaths, "", template.template()); + return allPaths; + } + + private static void gatherFieldTypes(Map paths, String pathToHere, Map template) { + for (var entry : template.entrySet()) { + var field = entry.getKey(); + var child = entry.getValue(); + var pathToChild = pathToHere.isEmpty() ? field : pathToHere + "." + field; + if (child instanceof Template.Object object) { + gatherFieldTypes(paths, pathToChild, object.children()); + } else { + var leaf = (Template.Leaf) child; + paths.put(pathToChild, leaf.type()); + } + } + } + void writeLogsDbMapping(XContentBuilder builder) throws IOException { builder.map(mapping.raw()); } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index 737fe5693215a..51e450c2a2da0 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -11,12 +11,16 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.datageneration.matchers.MatchResult; import org.elasticsearch.datageneration.matchers.Matcher; +import org.elasticsearch.datageneration.matchers.source.SourceTransforms; +import org.elasticsearch.datageneration.queries.QueryGenerator; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; @@ -37,6 +41,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.TreeMap; import static org.hamcrest.Matchers.equalTo; @@ -132,12 +137,52 @@ public void testMatchAllQuery() throws IOException { final MatchResult matchResult = Matcher.matchSource() .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) - .expected(getQueryHits(queryBaseline(searchSourceBuilder))) + .expected(getQueryHits(queryBaseline(searchSourceBuilder), true)) .ignoringSort(true) - .isEqualTo(getQueryHits(queryContender(searchSourceBuilder))); + .isEqualTo(getQueryHits(queryContender(searchSourceBuilder), true)); assertTrue(matchResult.getMessage(), matchResult.isMatch()); } + public void testRandomQueries() throws IOException { + int numberOfDocuments = ESTestCase.randomIntBetween(10, 50); + final List documents = generateDocuments(numberOfDocuments); + var mappingLookup = dataGenerationHelper.mapping().lookup(); + final List>> docsNormalized = documents.stream().map(d -> { + var document = XContentHelper.convertToMap(XContentType.JSON.xContent(), Strings.toString(d), true); + return SourceTransforms.normalize(document, mappingLookup); + }).toList(); + + indexDocuments(documents); + + QueryGenerator queryGenerator = new QueryGenerator(dataGenerationHelper.mapping()); + Map fieldsTypes = dataGenerationHelper.getTemplateFieldTypes(); + for (var e : fieldsTypes.entrySet()) { + var path = e.getKey(); + var type = e.getValue(); + var docsWithFields = docsNormalized.stream().filter(d -> d.containsKey(path)).toList(); + if (docsWithFields.isEmpty() == false) { + var doc = randomFrom(docsWithFields); + List values = doc.get(path).stream().filter(Objects::nonNull).toList(); + if (values.isEmpty() == false) { + Object value = randomFrom(values); + List queries = queryGenerator.generateQueries(type, path, value); + for (var query : queries) { + logger.info("Querying for field [{}] with value [{}]", path, value); + + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(query).size(numberOfDocuments); + final MatchResult matchResult = Matcher.matchSource() + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getQueryHits(queryBaseline(searchSourceBuilder), false)) + .ignoringSort(true) + .isEqualTo(getQueryHits(queryContender(searchSourceBuilder), false)); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + } + } + } + } + public void testTermsQuery() throws IOException { int numberOfDocuments = ESTestCase.randomIntBetween(20, 80); final List documents = generateDocuments(numberOfDocuments); @@ -150,9 +195,9 @@ public void testTermsQuery() throws IOException { final MatchResult matchResult = Matcher.matchSource() .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) - .expected(getQueryHits(queryBaseline(searchSourceBuilder))) + .expected(getQueryHits(queryBaseline(searchSourceBuilder), true)) .ignoringSort(true) - .isEqualTo(getQueryHits(queryContender(searchSourceBuilder))); + .isEqualTo(getQueryHits(queryContender(searchSourceBuilder), true)); assertTrue(matchResult.getMessage(), matchResult.isMatch()); } @@ -291,12 +336,15 @@ protected XContentBuilder generateDocument(final Instant timestamp) throws IOExc } @SuppressWarnings("unchecked") - private static List> getQueryHits(final Response response) throws IOException { + private static List> getQueryHits(final Response response, final boolean requireResults) throws IOException { final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); final Map hitsMap = (Map) map.get("hits"); final List> hitsList = (List>) hitsMap.get("hits"); - assertThat(hitsList.size(), greaterThan(0)); + + if (requireResults) { + assertThat(hitsList.size(), greaterThan(0)); + } return hitsList.stream() .sorted(Comparator.comparing((Map hit) -> ((String) hit.get("_id"))))