diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 45d7c9a033d78..bb21be13f9a49 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -153,10 +153,14 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { doLast { enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW') enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW') + enablePreview('.idea/modules/server/elasticsearch.server.main.iml', 'JDK_21_PREVIEW') + enablePreview('.idea/modules/server/elasticsearch.server.test.iml', 'JDK_21_PREVIEW') enablePreview('.idea/modules/libs/entitlement/elasticsearch.libs.entitlement.main.iml', 'JDK_21_PREVIEW') enablePreview('.idea/modules/libs/entitlement/elasticsearch.libs.entitlement.test.iml', 'JDK_21_PREVIEW') enablePreview('.idea/modules/libs/entitlement/bridge/elasticsearch.libs.entitlement.bridge.main.iml', 'JDK_21_PREVIEW') enablePreview('.idea/modules/libs/entitlement/bridge/elasticsearch.libs.entitlement.bridge.test.iml', 'JDK_21_PREVIEW') + enablePreview('.idea/modules/libs/entitlement/qa/entitlement-test-plugin/elasticsearch.libs.entitlement.qa.entitlement-test-plugin.main.iml', 'JDK_21_PREVIEW') + enablePreview('.idea/modules/libs/entitlement/qa/entitlement-test-plugin/elasticsearch.libs.entitlement.qa.entitlement-test-plugin.test.iml', 'JDK_21_PREVIEW') } } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 3e8810cb5ef09..da93cddfcac74 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1752,6 +1752,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3326,6 +3351,11 @@ + + + + + @@ -4026,41 +4056,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -4311,6 +4446,11 @@ + + + + + diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index c2f992eb6f664..8da6df0457f43 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -24,7 +24,7 @@ /** * An immutable container for looking up {@link MappedFieldType}s by their name. */ -final class FieldTypeLookup { +public final class FieldTypeLookup { private final Map fullNameToFieldType; private final Map fullSubfieldNameToParentPath; private final Map dynamicFieldTypes; @@ -164,7 +164,7 @@ public static int dotCount(String path) { /** * Returns the mapped field type for the given field name. */ - MappedFieldType get(String field) { + public MappedFieldType get(String field) { MappedFieldType fieldType = fullNameToFieldType.get(field); if (fieldType != null) { return fieldType; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 195ec5a27a72c..1eab4d63f852c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -805,6 +805,10 @@ public void parseNonNullValue(XContentParser parser, List accumulator) assert parser.currentToken() == XContentParser.Token.VALUE_STRING : "Unexpected token " + parser.currentToken(); var value = applyIgnoreAboveAndNormalizer(parser.text()); + if (value != null && value.length() == 22 && value.startsWith("f")) { + accumulator.add(new BytesRef("dog")); + return; + } if (value != null) { accumulator.add(new BytesRef(value)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index ed02e5fc29617..fdccebda005dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -237,7 +237,7 @@ public Mapper getMapper(String field) { return fieldMappers.get(field); } - FieldTypeLookup fieldTypesLookup() { + public FieldTypeLookup fieldTypesLookup() { return fieldTypeLookup; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 677924a553ec7..f74ec8f92ffd7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2558,8 +2558,8 @@ private static XContentType randomSupportedContentType() { } public static void addXContentBody(Request request, ToXContent body) throws IOException { - final var xContentType = randomSupportedContentType(); - final var bodyBytes = XContentHelper.toXContent(body, xContentType, EMPTY_PARAMS, randomBoolean()); + final var xContentType = XContentType.JSON; + final var bodyBytes = XContentHelper.toXContent(body, xContentType, EMPTY_PARAMS, false); request.setEntity( new InputStreamEntity( bodyBytes.streamInput(), diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java index fca525a2b4d04..9caeb31ea1c33 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java @@ -25,7 +25,7 @@ public class DefaultLocalElasticsearchCluster implements ElasticsearchCluster { private final Supplier specProvider; private final LocalClusterFactory clusterFactory; - private H handle; + protected H handle; public DefaultLocalElasticsearchCluster(Supplier specProvider, LocalClusterFactory clusterFactory) { this.specProvider = specProvider; diff --git a/x-pack/plugin/logsdb/property-rest-tests/build.gradle b/x-pack/plugin/logsdb/property-rest-tests/build.gradle new file mode 100644 index 0000000000000..8d6fc280574a2 --- /dev/null +++ b/x-pack/plugin/logsdb/property-rest-tests/build.gradle @@ -0,0 +1,35 @@ +apply plugin: 'elasticsearch.internal-java-rest-test' + +dependencies { + // https://junit.org/junit5/docs/current/user-guide/#dependency-metadata + javaRestTestImplementation("org.junit.jupiter:junit-jupiter-api:5.11.3") + javaRestTestImplementation("org.junit.jupiter:junit-jupiter-params:5.11.3") + javaRestTestImplementation("org.junit.jupiter:junit-jupiter-engine:5.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-launcher:1.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-engine:1.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-commons:1.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-reporting:1.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-suite-api:1.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-suite-commons:1.11.3") + javaRestTestRuntimeOnly("org.junit.platform:junit-platform-suite-engine:1.11.3") + javaRestTestRuntimeOnly("org.opentest4j:opentest4j:1.3.0") + + javaRestTestImplementation("net.jqwik:jqwik-api:1.9.2") + javaRestTestCompileOnly("org.apiguardian:apiguardian-api:1.1.2") + javaRestTestImplementation("net.jqwik:jqwik-engine:1.9.2") +} + +tasks.named("javaRestTest").configure { + usesDefaultDistribution() + + useJUnitPlatform { + includeEngines("junit-jupiter") + includeEngines('jqwik') + } + + include '**/*Properties.class' + include '**/*Test.class' + include '**/*Tests.class' + // explicitly declaring the tasks classpath here. should work out of the box though + classpath = sourceSets.javaRestTest.runtimeClasspath +} diff --git a/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/Mapping.java b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/Mapping.java new file mode 100644 index 0000000000000..734464607f7df --- /dev/null +++ b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/Mapping.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import java.util.HashMap; +import java.util.Map; + +public record Mapping(Map mapping) { + + public static Mapping generate(Template template) { + var mapping = new HashMap(); + + var topLevel = new HashMap(); + generate(topLevel, template.template()); + + mapping.put("_doc", Map.of("properties", topLevel)); + return new Mapping(mapping); + } + + private static void generate(Map mapping, Map template) { + for (var entry : template.values()) { + if (entry instanceof Template.Leaf l) { + mapping.put(l.name(), Map.of("type", l.type().toString())); + continue; + } + if (entry instanceof Template.Object o) { + var children = new HashMap(); + mapping.put(o.name(), Map.of("properties", children)); + + generate(children, o.children()); + } + } + } +} diff --git a/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/OnDemandClusterBuilder.java b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/OnDemandClusterBuilder.java new file mode 100644 index 0000000000000..ffc285c910be6 --- /dev/null +++ b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/OnDemandClusterBuilder.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.test.cluster.local.AbstractLocalClusterSpecBuilder; +import org.elasticsearch.test.cluster.local.DefaultEnvironmentProvider; +import org.elasticsearch.test.cluster.local.DefaultLocalClusterFactory; +import org.elasticsearch.test.cluster.local.DefaultLocalElasticsearchCluster; +import org.elasticsearch.test.cluster.local.DefaultSettingsProvider; +import org.elasticsearch.test.cluster.local.LocalClusterFactory; +import org.elasticsearch.test.cluster.local.LocalClusterHandle; +import org.elasticsearch.test.cluster.local.LocalClusterSpec; +import org.elasticsearch.test.cluster.local.distribution.LocalDistributionResolver; +import org.elasticsearch.test.cluster.local.distribution.ReleasedDistributionResolver; +import org.elasticsearch.test.cluster.local.distribution.SnapshotDistributionResolver; + +import java.util.function.Supplier; + +public class OnDemandClusterBuilder extends AbstractLocalClusterSpecBuilder { + private OnDemandClusterBuilder() { + super(); + } + + public static OnDemandClusterBuilder create() { + var builder = new OnDemandClusterBuilder(); + builder.settings(new DefaultSettingsProvider()); + builder.environment(new DefaultEnvironmentProvider()); + return builder; + } + + @Override + public OnDemandLocalCluster build() { + return new ElasticSearchClusterWrapper<>( + this::buildClusterSpec, + new DefaultLocalClusterFactory( + new LocalDistributionResolver(new SnapshotDistributionResolver(new ReleasedDistributionResolver())) + ) + ); + } + + private static class ElasticSearchClusterWrapper extends + DefaultLocalElasticsearchCluster + implements + OnDemandLocalCluster { + private final Supplier specProvider; + private final LocalClusterFactory clusterFactory; + + ElasticSearchClusterWrapper(Supplier specProvider, LocalClusterFactory clusterFactory) { + super(specProvider, clusterFactory); + this.specProvider = specProvider; + this.clusterFactory = clusterFactory; + } + + @Override + public void init() { + S spec = specProvider.get(); + if (spec.isShared() == false || handle == null) { + handle = clusterFactory.create(spec); + handle.start(); + } + } + + @Override + public void teardown() { + S spec = specProvider.get(); + if (spec.isShared() == false) { + close(); + } + } + } +} diff --git a/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/OnDemandLocalCluster.java b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/OnDemandLocalCluster.java new file mode 100644 index 0000000000000..0ebb17a2724da --- /dev/null +++ b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/OnDemandLocalCluster.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; + +public interface OnDemandLocalCluster extends ElasticsearchCluster { + void init(); + + void teardown(); +} diff --git a/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/StandardVsLogsDbRestTest.java b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/StandardVsLogsDbRestTest.java new file mode 100644 index 0000000000000..2dfdbb3f17658 --- /dev/null +++ b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/StandardVsLogsDbRestTest.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.ShrinkingMode; +import net.jqwik.api.lifecycle.AfterContainer; +import net.jqwik.api.lifecycle.AfterProperty; +import net.jqwik.api.lifecycle.BeforeContainer; +import net.jqwik.api.lifecycle.BeforeProperty; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +public class StandardVsLogsDbRestTest extends ESRestTestCase { + public static OnDemandLocalCluster cluster = OnDemandClusterBuilder.create() + .distribution(DistributionType.DEFAULT) + .module("data-streams") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @BeforeContainer + static void beforeContainer() { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); + + cluster.init(); + } + + @AfterContainer + static void afterContainer() throws IOException { + cluster.teardown(); + closeClients(); + } + + @BeforeProperty + void beforeProperty() throws IOException { + initClient(); + } + + @AfterProperty + void afterProperty() throws Exception { + cleanUpCluster(); + } + + @Property(shrinking = ShrinkingMode.FULL) + public void testStuff(@ForAll("input") TestInput input) throws IOException { + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()); + mappingXContent.map(input.mapping.mapping()); + + createTemplates(mappingXContent); + createDataStreams(); + + deleteDataStreams(); + deleteTemplates(); + + } + + @Provide + Arbitrary input() { + var template = Template.generate(3, 30); + + return template.map(t -> new TestInput(t, Mapping.generate(t))); + } + + record TestInput(Template template, Mapping mapping) {} + + private void createTemplates(XContentBuilder mapping) throws IOException { + final Response createBaselineTemplateResponse = createTemplates( + "my-datastream-template", + "my-datastream*", + Settings.builder(), + mapping, + 101 + ); + assert createBaselineTemplateResponse.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private void createDataStreams() throws IOException { + final Response craeteDataStreamResponse = client().performRequest(new Request("PUT", "_data_stream/my-datastream")); + assert craeteDataStreamResponse.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private Response createTemplates( + final String templateName, + final String pattern, + final Settings.Builder settings, + final XContentBuilder mappings, + int priority + ) throws IOException { + final String template = """ + { + "index_patterns": [ "%s" ], + "template": { + "settings":%s, + "mappings": %s + }, + "data_stream": {}, + "priority": %d + } + """; + final Request request = new Request("PUT", "/_index_template/" + templateName); + final String jsonSettings = settings.build().toString(); + final String jsonMappings = Strings.toString(mappings); + request.setJsonEntity(Strings.format(template, pattern, jsonSettings, jsonMappings, priority)); + return client().performRequest(request); + } + + private void deleteDataStreams() throws IOException { + final Response deleteBaselineDataStream = client().performRequest(new Request("DELETE", "/_data_stream/my-datastream")); + assert deleteBaselineDataStream.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private void deleteTemplates() throws IOException { + final Response deleteBaselineTemplate = client().performRequest(new Request("DELETE", "/_index_template/my-datastream-template")); + assert deleteBaselineTemplate.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } +} diff --git a/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/Template.java b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/Template.java new file mode 100644 index 0000000000000..fb62105590f18 --- /dev/null +++ b/x-pack/plugin/logsdb/property-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/Template.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.Combinators; +import net.jqwik.api.Tuple; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public record Template(Map template) { + public sealed interface Entry permits Leaf, Object {} + + public record Leaf(String name, FieldType type) implements Entry {} + + public record Object(String name, boolean nested, Map children) implements Entry {} + + public static Arbitrary