Add Dynamic Properties Support

CHANGELOG.md

@@ -23,8 +23,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503))
 - Add ref_path support for package-based hunspell dictionary loading ([#20840](https://github.com/opensearch-project/OpenSearch/pull/20840))
 - Add support for enabling pluggable data formats, starting with phase-1 of decoupling shard from engine, and introducing basic abstractions ([#20675](https://github.com/opensearch-project/OpenSearch/pull/20675))
-
 - Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526))
+- Add `dynamic_properties` for pattern-based field mappings; matched fields are materialized without a cluster-state mapping update ([#20816](https://github.com/opensearch-project/OpenSearch/pull/20816))
+
 ### Changed
 - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
 - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570))

server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java

@@ -160,7 +160,11 @@ public DocumentMapper(MapperService mapperService, Mapping mapping) {
         this.documentParser = new DocumentParser(indexSettings, mapperService.documentMapperParser(), this);
 
         final IndexAnalyzers indexAnalyzers = mapperService.getIndexAnalyzers();
-        this.fieldMappers = MappingLookup.fromMapping(this.mapping, indexAnalyzers.getDefaultIndexAnalyzer());
+        this.fieldMappers = MappingLookup.fromMapping(
+            this.mapping,
+            indexAnalyzers.getDefaultIndexAnalyzer(),
+            mapperService.documentMapperParser()
+        );
 
         try {
             mappingSource = new CompressedXContent(this, ToXContent.EMPTY_PARAMS);

server/src/main/java/org/opensearch/index/mapper/DocumentParser.java

@@ -56,6 +56,7 @@
 import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
@@ -1323,6 +1324,50 @@ private static void parseDynamicValue(
         if (dynamic == ObjectMapper.Dynamic.FALSE) {
             return;
         }
+
+        // If a dynamic_property matches this field, use it without updating the index mapping (no cluster state update;
+        // see DynamicProperty javadoc for authorization/audit implications).
+        String fullPath = parentMapper.fullPath().isEmpty() ? currentFieldName : parentMapper.fullPath() + "." + currentFieldName;
+        DynamicProperty dynamicProperty = context.root().findDynamicProperty(fullPath);
+        if (dynamicProperty != null) {
+            Mapper cached = context.lookupDynamicPropertyMapper(fullPath);
+            if (cached != null) {
+                parseObjectOrField(context, cached);
+                return;
+            }
+            Map<String, Object> config = new HashMap<>(dynamicProperty.mappingForName(currentFieldName));
+            Object typeNode = config.get("type");
+            if (typeNode == null) {
+                throw new MapperParsingException(
+                    "dynamic_property pattern ["
+                        + dynamicProperty.getPattern()
+                        + "] matched field ["
+                        + fullPath
+                        + "] but its mapping has no [type]"
+                );
+            }
+            String type = typeNode.toString();
+            Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext();
+            Mapper.TypeParser typeParser = parserContext.typeParser(type);
+            if (typeParser == null) {
+                throw new MapperParsingException(
+                    "No handler for type ["
+                        + type
+                        + "] in dynamic_property pattern ["
+                        + dynamicProperty.getPattern()
+                        + "] for field ["
+                        + fullPath
+                        + "]"
+                );
+            }
+            Mapper.Builder<?> builder = typeParser.parse(currentFieldName, config, parserContext);
+            Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
+            Mapper mapper = builder.build(builderContext);
+            context.rememberDynamicPropertyMapper(fullPath, mapper);
+            parseObjectOrField(context, mapper);
+            return;
+        }
+
         final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
         if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES && builder == null) {
             // For FALSE_ALLOW_TEMPLATES, if no template matches, we still need to consume the token

server/src/main/java/org/opensearch/index/mapper/DynamicProperty.java

@@ -0,0 +1,112 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.regex.Regex;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * A dynamic property defines a pattern-based mapping: any field whose name matches
+ * the pattern is mapped using the same configuration without updating the index mapping.
+ * Similar to Solr dynamic fields; avoids cluster state updates for high-throughput ingestion.
+ *
+ * <p><b>Operational note:</b> When a document field matches a dynamic property, ingest builds
+ * {@link FieldMapper} instances and indexes using that configuration <em>without</em> persisting
+ * new field definitions into the cluster state mapping (no mapping-update round trip for that
+ * field). That is intentional for performance and matches Solr-style dynamic fields.
+ *
+ * <p>Downstream effects: anything that runs only when the cluster state mapping is updated
+ * (for example authorization checks, audit hooks, or plugins that observe {@code PUT mapping}
+ * / dynamic mapping updates) will not run for fields materialized solely through dynamic
+ * property matching. Operators and integrators should not assume those hooks fire per matched
+ * field. Use explicit {@code properties} in the index mapping if a field must go through the
+ * normal mapping-update pipeline.
+ *
+ * <p>Index settings may not list {@code dynamic_properties} with only a catch-all {@code *} pattern;
+ * at least one more specific pattern is required so routing intent is explicit. To default-type every
+ * unknown field (e.g. keyword) without that constraint, use {@code dynamic_templates} on the root
+ * mapping instead.
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "3.0.0")
+public class DynamicProperty implements ToXContentObject {
+
+    private final String pattern;
+    private final Map<String, Object> mapping;
+
+    public DynamicProperty(String pattern, Map<String, Object> mapping) {
+        this.pattern = pattern;
+        this.mapping = mapping;
+    }
+
+    /** Returns the wildcard pattern (e.g. {@code *_i}, {@code *}). */
+    public String getPattern() {
+        return pattern;
+    }
+
+    /** Returns the mapping configuration for matching fields (e.g. type and options). */
+    public Map<String, Object> getMapping() {
+        return mapping;
+    }
+
+    /**
+     * Returns whether the given name matches this dynamic property's pattern.
+     * At ingest and query, the name is the <strong>full dotted path</strong> from the mapping root
+     * (e.g. {@code parent.count_i}), not only the leaf segment, so patterns may include {@code .} in
+     * the matched text. Uses the same simple wildcard matching as dynamic_templates.
+     */
+    public boolean matches(String fieldName) {
+        return Regex.simpleMatch(pattern, fieldName);
+    }
+
+    /**
+     * Returns a copy of the mapping suitable for parsing a concrete field with the given name.
+     * Supports {name} placeholder in mapping values.
+     */
+    public Map<String, Object> mappingForName(String fieldName) {
+        return processMap(mapping, fieldName);
+    }
+
+    private Map<String, Object> processMap(Map<String, Object> map, String fieldName) {
+        Map<String, Object> result = new TreeMap<>();
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            String key = entry.getKey().replace("{name}", fieldName);
+            Object value = entry.getValue();
+            if (value instanceof Map<?, ?> rawNested) {
+                Map<String, Object> nested = new TreeMap<>();
+                for (Map.Entry<?, ?> e : rawNested.entrySet()) {
+                    if ((e.getKey() instanceof String) == false) {
+                        throw new IllegalArgumentException(
+                            "dynamic_property nested mapping must use string keys; got [" + e.getKey() + "]"
+                        );
+                    }
+                    nested.put((String) e.getKey(), e.getValue());
+                }
+                value = processMap(nested, fieldName);
+            } else if (value instanceof String) {
+                value = value.toString().replace("{name}", fieldName);
+            }
+            result.put(key, value);
+        }
+        return result;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(pattern, new TreeMap<>(mapping));
+        return builder;
+    }
+}

server/src/main/java/org/opensearch/index/mapper/DynamicPropertyFieldTypeResolver.java

@@ -0,0 +1,112 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Resolves {@link MappedFieldType} for field names that match {@link RootObjectMapper#dynamicProperties()}
+ * without adding those fields to the serialized mapping. Query-time resolution mirrors ingest-time
+ * {@link DynamicProperty} handling: no cluster state mapping update occurs when a name matches.
+ *
+ * <p>Callers that rely on mapping-update side effects (security, audit, etc.) should see
+ * {@link DynamicProperty} class documentation.
+ *
+ * <p><b>Cache invalidation:</b> Each {@link DocumentMapper} constructor runs
+ * {@link MappingLookup#fromMapping} and constructs a new resolver with an empty cache.
+ * {@link DocumentMapper#merge} always returns a new {@link DocumentMapper} for the merged
+ * {@link Mapping}, so {@link RootObjectMapper} / {@code dynamic_properties} updates never mutate
+ * this instance in place; previous mappers (and their caches) are simply replaced on
+ * {@link MapperService}. No explicit cache clear is required.
+ *
+ * @opensearch.internal
+ */
+final class DynamicPropertyFieldTypeResolver {
+
+    private static final int MAX_RESOLVER_CACHE_SIZE = 1024;
+
+    private final RootObjectMapper root;
+    private final DocumentMapperParser documentMapperParser;
+    /** LRU-bounded: high-cardinality field names matching patterns must not grow memory without bound. */
+    private final Map<String, MappedFieldType> cache = new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, MappedFieldType> eldest) {
+            return size() > MAX_RESOLVER_CACHE_SIZE;
+        }
+    };
+
+    DynamicPropertyFieldTypeResolver(RootObjectMapper root, DocumentMapperParser documentMapperParser) {
+        this.root = root;
+        this.documentMapperParser = documentMapperParser;
+    }
+
+    /**
+     * Returns a field type for the given full path, or null if no dynamic_property applies.
+     */
+    synchronized MappedFieldType resolve(String fullFieldName) {
+        if (root.dynamicProperties().length == 0) {
+            return null;
+        }
+        MappedFieldType cached = cache.get(fullFieldName);
+        if (cached != null) {
+            return cached;
+        }
+        DynamicProperty dp = root.findDynamicProperty(fullFieldName);
+        if (dp == null) {
+            return null;
+        }
+        String leaf = leafName(fullFieldName);
+        Map<String, Object> config = new HashMap<>(dp.mappingForName(leaf));
+        Object typeNode = config.get("type");
+        if (typeNode == null) {
+            return null;
+        }
+        String type = typeNode.toString();
+        Mapper.TypeParser.ParserContext parserContext = documentMapperParser.parserContext();
+        Mapper.TypeParser typeParser = parserContext.typeParser(type);
+        if (typeParser == null) {
+            return null;
+        }
+        Mapper.Builder<?> builder = typeParser.parse(leaf, config, parserContext);
+        ContentPath path = parentPath(fullFieldName);
+        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(
+            documentMapperParser.mapperService.getIndexSettings().getSettings(),
+            path
+        );
+        Mapper mapper = builder.build(builderContext);
+        if ((mapper instanceof FieldMapper) == false) {
+            return null;
+        }
+        MappedFieldType fieldType = ((FieldMapper) mapper).fieldType();
+        cache.put(fullFieldName, fieldType);
+        return fieldType;
+    }
+
+    /** Last path segment; used for {@link DynamicProperty#mappingForName} and type parser simple name. */
+    private static String leafName(String fullFieldName) {
+        int lastDot = fullFieldName.lastIndexOf('.');
+        return lastDot < 0 ? fullFieldName : fullFieldName.substring(lastDot + 1);
+    }
+
+    /** Content path for all parent segments (empty at root). */
+    private static ContentPath parentPath(String fullFieldName) {
+        ContentPath path = new ContentPath();
+        int lastDot = fullFieldName.lastIndexOf('.');
+        if (lastDot <= 0) {
+            return path;
+        }
+        String prefix = fullFieldName.substring(0, lastDot);
+        for (String segment : prefix.split("\\.")) {
+            path.add(segment);
+        }
+        return path;
+    }
+}

server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java

@@ -65,11 +65,21 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
     private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
     private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
 
+    private final DynamicPropertyFieldTypeResolver dynamicPropertyFieldTypes;
+
     FieldTypeLookup() {
         this(Collections.emptyList(), Collections.emptyList());
     }
 
     FieldTypeLookup(Collection<FieldMapper> fieldMappers, Collection<FieldAliasMapper> fieldAliasMappers) {
+        this(fieldMappers, fieldAliasMappers, null);
+    }
+
+    FieldTypeLookup(
+        Collection<FieldMapper> fieldMappers,
+        Collection<FieldAliasMapper> fieldAliasMappers,
+        DynamicPropertyFieldTypeResolver dynamicPropertyFieldTypes
+    ) {
         Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
 
         for (FieldMapper fieldMapper : fieldMappers) {
@@ -98,6 +108,7 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
         }
 
         this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
+        this.dynamicPropertyFieldTypes = dynamicPropertyFieldTypes;
     }
 
     /**
@@ -112,7 +123,15 @@ public MappedFieldType get(String field) {
 
         // If the mapping contains fields that support dynamic sub-key lookup, check
         // if this could correspond to a keyed field of the form 'path_to_field.path_to_key'.
-        return dynamicKeyLookup.get(field);
+        fieldType = dynamicKeyLookup.get(field);
+        if (fieldType != null) {
+            return fieldType;
+        }
+
+        if (dynamicPropertyFieldTypes != null) {
+            return dynamicPropertyFieldTypes.resolve(field);
+        }
+        return null;
     }
 
     /**