diff --git a/test/framework/build.gradle b/test/framework/build.gradle index c7e08eb3cdfa9..3a2f4037f702d 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -33,6 +33,7 @@ dependencies { api "org.elasticsearch:mocksocket:${versions.mocksocket}" testImplementation project(':x-pack:plugin:mapper-unsigned-long') + testImplementation project(':x-pack:plugin:mapper-counted-keyword') testImplementation project(":modules:mapper-extras") } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 96b75f29382e2..4bf65fcf6ecf6 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -11,6 +11,7 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator; @@ -34,7 +35,8 @@ public enum FieldType { DOUBLE("double"), FLOAT("float"), HALF_FLOAT("half_float"), - SCALED_FLOAT("scaled_float"); + SCALED_FLOAT("scaled_float"), + COUNTED_KEYWORD("counted_keyword"); private final String name; @@ -54,6 +56,7 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) { case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource); case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource); case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource); + case COUNTED_KEYWORD -> new CountedKeywordFieldDataGenerator(fieldName, dataSource); }; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index df28282fca407..3aad8e55c326c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -54,6 +54,10 @@ default DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper re return null; } + default DataSourceResponse.RepeatingWrapper handle(DataSourceRequest.RepeatingWrapper request) { + return null; + } + default DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index d77925f097b5a..999e9a8bf6c7b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -85,6 +85,12 @@ public DataSourceResponse.ArrayWrapper accept(DataSourceHandler handler) { } } + record RepeatingWrapper() implements DataSourceRequest { + public DataSourceResponse.RepeatingWrapper accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record ChildFieldGenerator(DataGeneratorSpecification specification) implements DataSourceRequest { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index fa8f56b3e071b..3722e75d916dc 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -39,6 +39,8 @@ record NullWrapper(Function, Supplier> wrapper) impleme record ArrayWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} + record RepeatingWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} + interface ChildFieldGenerator extends DataSourceResponse { int generateChildFieldCount(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 04cb9467270d4..2567037488f3f 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -33,6 +33,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques case KEYWORD -> keywordMapping(request, map); case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> plain(map); case SCALED_FLOAT -> scaledFloatMapping(map); + case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean())); }); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java index 8af26c28ef5b3..3640f98e25b55 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java @@ -11,6 +11,7 @@ import org.elasticsearch.test.ESTestCase; +import java.util.HashSet; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.IntStream; @@ -26,6 +27,11 @@ public DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper ign return new DataSourceResponse.ArrayWrapper(wrapInArray()); } + @Override + public DataSourceResponse.RepeatingWrapper handle(DataSourceRequest.RepeatingWrapper ignored) { + return new DataSourceResponse.RepeatingWrapper(repeatValues()); + } + private static Function, Supplier> injectNulls() { // Inject some nulls but majority of data should be non-null (as it likely is in reality). return (values) -> () -> ESTestCase.randomDouble() <= 0.05 ? null : values.get(); @@ -41,4 +47,19 @@ private static Function, Supplier> wrapInArray() { return values.get(); }; } + + private static Function, Supplier> repeatValues() { + return (values) -> { + HashSet previousValues = new HashSet<>(); + return () -> { + if (previousValues.size() > 0 && ESTestCase.randomBoolean()) { + return ESTestCase.randomFrom(previousValues); + } else { + var value = values.get(); + previousValues.add(value); + return value; + } + }; + }; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java new file mode 100644 index 0000000000000..f09ef2397bfc7 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.fields.leaf; + +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.util.HashSet; +import java.util.Set; +import java.util.function.Supplier; + +public class CountedKeywordFieldDataGenerator implements FieldDataGenerator { + private final Supplier valueGenerator; + private final Set previousStrings = new HashSet<>(); + + public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource) { + var strings = dataSource.get(new DataSourceRequest.StringGenerator()); + var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); + var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var repeats = dataSource.get(new DataSourceRequest.RepeatingWrapper()); + + this.valueGenerator = arrays.wrapper().compose(nulls.wrapper().compose(repeats.wrapper())).apply(() -> strings.generator().get()); + } + + @Override + public Object generateValue() { + return valueGenerator.get(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index 960cc38e55c82..f86eb31f47cc6 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -15,6 +15,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.math.BigInteger; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -223,4 +224,68 @@ private static BigInteger toBigInteger(Object value) { return (BigInteger) value; } } + + class CountedKeywordMatcher implements FieldSpecificMatcher { + private final XContentBuilder actualMappings; + private final Settings.Builder actualSettings; + private final XContentBuilder expectedMappings; + private final Settings.Builder expectedSettings; + + CountedKeywordMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + } + + private static List normalize(List values) { + return values.stream().filter(Objects::nonNull).map(it -> (String) it).toList(); + } + + private static boolean matchCountsEqualExact(List actualNormalized, List expectedNormalized) { + HashMap counts = new HashMap<>(); + for (String value : actualNormalized) { + counts.put(value, counts.getOrDefault(value, 0) + 1); + } + for (String value : expectedNormalized) { + int newCount = counts.getOrDefault(value, 0) - 1; + if (newCount == 0) { + counts.remove(value); + } else { + counts.put(value, newCount); + } + } + + return counts.isEmpty(); + } + + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + var actualNormalized = normalize(actual); + var expectedNormalized = normalize(expected); + + return matchCountsEqualExact(actualNormalized, expectedNormalized) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of type [counted_keyword] don't match after normalization, normalized" + + prettyPrintCollections(actualNormalized, expectedNormalized) + ) + ); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java index d58d081e7c9f9..96b8824b76af3 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java @@ -57,7 +57,9 @@ public SourceMatcher( "scaled_float", new FieldSpecificMatcher.ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), "unsigned_long", - new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), + "counted_keyword", + new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); } @@ -100,17 +102,8 @@ private MatchResult compareSource(Map> actual, Map matchWithGenericMatcher(actualValues, expectedValues) ); if (matchIncludingFieldSpecificMatchers.isMatch() == false) { var message = "Source documents don't match for field [" + name + "]: " + matchIncludingFieldSpecificMatchers.getMessage(); diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java index d9750328ff3fa..f5ba8bd02fa88 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; import java.io.IOException; @@ -110,7 +111,7 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG var mappingService = new MapperServiceTestCase() { @Override protected Collection getPlugins() { - return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin()); + return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin(), new CountedKeywordMapperPlugin()); } }.createMapperService(mappingXContent); diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java index 74f70bae4d0c1..bc7a558a0b687 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java @@ -105,4 +105,38 @@ public void testMappedMismatch() throws IOException { var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertFalse(sut.match().isMatch()); } + + public void testCountedKeywordMatch() throws IOException { + List> actual = List.of(Map.of("field", List.of("a", "b", "a", "c", "b", "a"))); + List> expected = List.of(Map.of("field", List.of("a", "b", "a", "c", "b", "a"))); + + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); + mapping.startObject(); + mapping.startObject("_doc"); + { + mapping.startObject("field").field("type", "counted_keyword").endObject(); + } + mapping.endObject(); + mapping.endObject(); + + var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + assertTrue(sut.match().isMatch()); + } + + public void testCountedKeywordMismatch() throws IOException { + List> actual = List.of(Map.of("field", List.of("a", "b", "a", "c", "b", "a"))); + List> expected = List.of(Map.of("field", List.of("a", "b", "c", "a"))); + + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); + mapping.startObject(); + mapping.startObject("_doc"); + { + mapping.startObject("field").field("type", "counted_keyword").endObject(); + } + mapping.endObject(); + mapping.endObject(); + + var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + assertFalse(sut.match().isMatch()); + } }