diff --git a/.gitignore b/.gitignore index cac5a799012e1..f72dde1cac3d1 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,12 @@ server/src/main/resources/transport/defined/manifest.txt # JEnv .java-version + +# Claude Flow and Swarm directories +.claude-flow/ +.swarm/ + +# Temporary test files +test-*.java +bug-*.json +*-bug-analysis.md diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/60_types_filter_objects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/60_types_filter_objects.yml new file mode 100644 index 0000000000000..1592e233ff0f5 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/60_types_filter_objects.yml @@ -0,0 +1,165 @@ +--- +setup: + - requires: + cluster_features: "gte_v8.2.0" + reason: Field type filters were added in 8.2 + - do: + indices.create: + index: test_types_filter + body: + mappings: + properties: + product: + type: object + properties: + name: + type: text + price: + type: float + tags: + type: keyword + category: + type: object + properties: + title: + type: text + id: + type: integer + description: + type: text + nested_comments: + type: nested + properties: + text: + type: text + rating: + type: integer + +--- +"Types filter excludes object fields when not requested": + - do: + field_caps: + index: 'test_types_filter' + fields: '*' + types: 'text' + + # Should include text fields + - is_true: fields.product\\.name + - is_true: fields.category\\.title + - is_true: fields.description + - is_true: fields.nested_comments\\.text + + # Should NOT include object or nested fields + - is_false: fields.product + - is_false: fields.category + - is_false: fields.nested_comments + + # Should NOT include non-text fields + - is_false: fields.product\\.price + - is_false: fields.product\\.tags + - is_false: fields.category\\.id + - is_false: fields.nested_comments\\.rating + +--- +"Types filter includes object fields when explicitly requested": + - do: + field_caps: + index: 'test_types_filter' + fields: '*' + types: 'text,object' + + # Should include text fields + - is_true: fields.product\\.name + - is_true: fields.category\\.title + - is_true: fields.description + + # Should include object fields (but not nested) + - is_true: fields.product + - is_true: fields.category + + # Should NOT include nested fields (not in types filter) + - is_false: fields.nested_comments + + # Should NOT include other types + - is_false: fields.product\\.price + - is_false: fields.product\\.tags + +--- +"Types filter includes nested fields when explicitly requested": + - do: + field_caps: + index: 'test_types_filter' + fields: '*' + types: 'nested,text' + + # Should include text fields + - is_true: fields.product\\.name + - is_true: fields.nested_comments\\.text + + # Should include nested fields + - is_true: fields.nested_comments + + # Should NOT include object fields (not in types filter) + - is_false: fields.product + - is_false: fields.category + +--- +"Types filter with keyword type only": + - do: + field_caps: + index: 'test_types_filter' + fields: '*' + types: 'keyword' + + # Should include keyword field + - is_true: fields.product\\.tags + + # Should NOT include any object/nested fields + - is_false: fields.product + - is_false: fields.category + - is_false: fields.nested_comments + + # Should NOT include other types + - is_false: fields.product\\.name + - is_false: fields.product\\.price + - is_false: fields.description + +--- +"No types filter includes all fields": + - do: + field_caps: + index: 'test_types_filter' + fields: '*' + + # Should include all fields + - is_true: fields.product + - is_true: fields.product\\.name + - is_true: fields.product\\.price + - is_true: fields.product\\.tags + - is_true: fields.category + - is_true: fields.category\\.title + - is_true: fields.category\\.id + - is_true: fields.description + - is_true: fields.nested_comments + - is_true: fields.nested_comments\\.text + - is_true: fields.nested_comments\\.rating + +--- +"Types filter combined with -parent filter": + - do: + field_caps: + index: 'test_types_filter' + fields: '*' + types: 'text,object' + filters: '-parent' + + # Should include text fields + - is_true: fields.product\\.name + - is_true: fields.category\\.title + - is_true: fields.description + + # Should NOT include parent objects even though object is in types filter + # because -parent filter takes precedence + - is_false: fields.product + - is_false: fields.category + - is_false: fields.nested_comments \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index dc73be9ed7559..cacc7dfa4b762 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -206,17 +206,27 @@ static Map retrieveFieldCaps( if (context.getFieldType(parentField) == null) { // no field type, it must be an object field String type = context.nestedLookup().getNestedMappers().get(parentField) != null ? "nested" : "object"; - IndexFieldCapabilities fieldCap = new IndexFieldCapabilities( - parentField, - type, - false, - false, - false, - false, - null, - Map.of() - ); - responseMap.put(parentField, fieldCap); + + // Apply types filter to parent object fields + boolean shouldInclude = true; + if (types != null && types.length > 0) { + Set acceptedTypes = Set.of(types); + shouldInclude = acceptedTypes.contains(type); + } + + if (shouldInclude) { + IndexFieldCapabilities fieldCap = new IndexFieldCapabilities( + parentField, + type, + false, + false, + false, + false, + null, + Map.of() + ); + responseMap.put(parentField, fieldCap); + } } dotIndex = parentField.lastIndexOf('.'); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a43575b8f990c..73d5ae6debc68 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -332,6 +332,10 @@ public final void validate(MappingLookup mappers) { throw new IllegalArgumentException("[copy_to] may not be used to copy from a multi-field: [" + this.fullPath() + "]"); } + // Check if dynamic mappings are disabled + ObjectMapper.Dynamic rootDynamic = ObjectMapper.Dynamic.getRootDynamic(mappers); + boolean isDynamicDisabled = rootDynamic == ObjectMapper.Dynamic.FALSE; + final String sourceScope = mappers.nestedLookup().getNestedParent(this.fullPath()); for (String copyTo : this.copyTo().copyToFields()) { if (mappers.isMultiField(copyTo)) { @@ -341,6 +345,13 @@ public final void validate(MappingLookup mappers) { throw new IllegalArgumentException("Cannot copy to field [" + copyTo + "] since it is mapped as an object"); } + // When dynamic is false, check if the target field exists + if (isDynamicDisabled && mappers.getMapper(copyTo) == null) { + throw new IllegalArgumentException( + "Cannot copy to field [" + copyTo + "] because it does not exist and dynamic mappings are disabled" + ); + } + final String targetScope = mappers.nestedLookup().getNestedParent(copyTo); checkNestedScopeCompatibility(sourceScope, targetScope); } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTypesFilterTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTypesFilterTests.java new file mode 100644 index 0000000000000..4d09bb78c4519 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTypesFilterTests.java @@ -0,0 +1,284 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.fieldcaps; + +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.plugins.FieldPredicate; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Map; +import java.util.function.Predicate; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FieldCapabilitiesTypesFilterTests extends MapperServiceTestCase { + + public void testTypesFilterExcludesObjectFields() throws IOException { + // Create a mapping with an object field containing text and numeric fields + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("properties"); + { + b.startObject("product"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("name"); + b.field("type", "text"); + b.endObject(); + + b.startObject("price"); + b.field("type", "float"); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + + b.startObject("description"); + b.field("type", "text"); + b.endObject(); + } + b.endObject(); + })); + + SearchExecutionContext context = createSearchExecutionContext(mapperService); + + // Mock IndexShard for field info + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.getFieldInfos()).thenReturn(null); + + // Test with types filter for "text" only + Predicate fieldNameFilter = field -> true; + String[] filters = {}; + String[] types = {"text"}; + FieldPredicate fieldPredicate = FieldPredicate.ACCEPT_ALL; + + Map result = FieldCapabilitiesFetcher.retrieveFieldCaps( + context, + fieldNameFilter, + filters, + types, + fieldPredicate, + indexShard, + true + ); + + // Should include text fields + assertTrue("Should include product.name (text field)", result.containsKey("product.name")); + assertTrue("Should include description (text field)", result.containsKey("description")); + + // Should NOT include object field + assertFalse("Should NOT include product (object field) when filtering for text types", result.containsKey("product")); + + // Should NOT include float field + assertFalse("Should NOT include product.price (float field)", result.containsKey("product.price")); + } + + public void testTypesFilterIncludesObjectWhenRequested() throws IOException { + // Create a mapping with an object field + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("properties"); + { + b.startObject("product"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("name"); + b.field("type", "text"); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + SearchExecutionContext context = createSearchExecutionContext(mapperService); + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.getFieldInfos()).thenReturn(null); + + // Test with types filter including "object" + Predicate fieldNameFilter = field -> true; + String[] filters = {}; + String[] types = {"text", "object"}; + FieldPredicate fieldPredicate = FieldPredicate.ACCEPT_ALL; + + Map result = FieldCapabilitiesFetcher.retrieveFieldCaps( + context, + fieldNameFilter, + filters, + types, + fieldPredicate, + indexShard, + true + ); + + // Should include both text and object fields + assertTrue("Should include product.name (text field)", result.containsKey("product.name")); + assertTrue("Should include product (object field) when object is in types filter", result.containsKey("product")); + } + + public void testTypesFilterWithNestedFields() throws IOException { + // Create a mapping with nested fields + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("properties"); + { + b.startObject("comments"); + { + b.field("type", "nested"); + b.startObject("properties"); + { + b.startObject("text"); + b.field("type", "text"); + b.endObject(); + + b.startObject("rating"); + b.field("type", "integer"); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + SearchExecutionContext context = createSearchExecutionContext(mapperService); + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.getFieldInfos()).thenReturn(null); + + // Test with types filter for "text" only + Predicate fieldNameFilter = field -> true; + String[] filters = {}; + String[] types = {"text"}; + FieldPredicate fieldPredicate = FieldPredicate.ACCEPT_ALL; + + Map result = FieldCapabilitiesFetcher.retrieveFieldCaps( + context, + fieldNameFilter, + filters, + types, + fieldPredicate, + indexShard, + true + ); + + // Should include text field + assertTrue("Should include comments.text (text field)", result.containsKey("comments.text")); + + // Should NOT include nested field when filtering for text + assertFalse("Should NOT include comments (nested field) when filtering for text types", result.containsKey("comments")); + + // Should NOT include integer field + assertFalse("Should NOT include comments.rating (integer field)", result.containsKey("comments.rating")); + } + + public void testNoTypesFilterIncludesAllFields() throws IOException { + // Create a mapping with mixed field types + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("properties"); + { + b.startObject("product"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("name"); + b.field("type", "text"); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + SearchExecutionContext context = createSearchExecutionContext(mapperService); + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.getFieldInfos()).thenReturn(null); + + // Test with no types filter (empty array) + Predicate fieldNameFilter = field -> true; + String[] filters = {}; + String[] types = {}; // No types filter + FieldPredicate fieldPredicate = FieldPredicate.ACCEPT_ALL; + + Map result = FieldCapabilitiesFetcher.retrieveFieldCaps( + context, + fieldNameFilter, + filters, + types, + fieldPredicate, + indexShard, + true + ); + + // Should include all fields when no types filter + assertTrue("Should include product.name when no types filter", result.containsKey("product.name")); + assertTrue("Should include product (object) when no types filter", result.containsKey("product")); + } + + public void testTypesFilterWithExcludeParentFilter() throws IOException { + // Create a mapping with object fields + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("properties"); + { + b.startObject("product"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("name"); + b.field("type", "text"); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + SearchExecutionContext context = createSearchExecutionContext(mapperService); + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.getFieldInfos()).thenReturn(null); + + // Test with types filter and -parent filter combined + Predicate fieldNameFilter = field -> true; + String[] filters = {"-parent"}; // Exclude parent objects + String[] types = {"text", "object"}; // Include both text and object types + FieldPredicate fieldPredicate = FieldPredicate.ACCEPT_ALL; + + Map result = FieldCapabilitiesFetcher.retrieveFieldCaps( + context, + fieldNameFilter, + filters, + types, + fieldPredicate, + indexShard, + true + ); + + // Should include text field + assertTrue("Should include product.name (text field)", result.containsKey("product.name")); + + // Should NOT include object field even though it's in types filter, because -parent overrides + assertFalse("Should NOT include product when -parent filter is used", result.containsKey("product")); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java index aa184ddf465d5..e4f6f07a5d00c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java @@ -32,6 +32,62 @@ public class CopyToMapperTests extends MapperServiceTestCase { + public void testCopyToNonExistentFieldWithDynamicFalse() { + Exception e = expectThrows(IllegalArgumentException.class, () -> createDocumentMapper(topMapping(b -> { + b.field("dynamic", false); + b.startObject("properties"); + { + b.startObject("test_field"); + { + b.field("type", "text"); + b.field("copy_to", "missing_field"); + } + b.endObject(); + } + b.endObject(); + }))); + assertThat( + e.getMessage(), + equalTo("Cannot copy to field [missing_field] because it does not exist and dynamic mappings are disabled") + ); + } + + public void testCopyToExistingFieldWithDynamicFalse() throws Exception { + // This should succeed as the target field exists + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", false); + b.startObject("properties"); + { + b.startObject("test_field"); + { + b.field("type", "text"); + b.field("copy_to", "target_field"); + } + b.endObject(); + b.startObject("target_field"); + { + b.field("type", "text"); + } + b.endObject(); + } + b.endObject(); + })); + assertNotNull(mapper); + } + + public void testCopyToNonExistentFieldWithDynamicTrue() throws Exception { + // This should succeed as dynamic is true (default) + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("test_field"); + { + b.field("type", "text"); + b.field("copy_to", "missing_field"); + } + b.endObject(); + })); + assertNotNull(mapperService.documentMapper()); + } + @SuppressWarnings("unchecked") public void testCopyToFieldsParsing() throws Exception {