OTLP: store units in mappings

docs/changelog/134709.yaml

@@ -0,0 +1,5 @@
+pr: 134709
+summary: "OTLP: store units in mappings"
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java

@@ -68,6 +68,7 @@ public final class BulkRequestParser {
     private static final ParseField REQUIRE_DATA_STREAM = new ParseField(DocWriteRequest.REQUIRE_DATA_STREAM);
     private static final ParseField LIST_EXECUTED_PIPELINES = new ParseField(DocWriteRequest.LIST_EXECUTED_PIPELINES);
     private static final ParseField DYNAMIC_TEMPLATES = new ParseField("dynamic_templates");
+    private static final ParseField DYNAMIC_TEMPLATES_PARAMS = new ParseField("dynamic_templates_params");
 
     // TODO: Remove this parameter once the BulkMonitoring endpoint has been removed
     // for CompatibleApi V7 this means to deprecate on type, for V8+ it means to throw an error
@@ -359,6 +360,7 @@ private boolean parseActionLine(BytesReference data, int from, int to) throws IO
                 boolean requireAlias = defaultRequireAlias != null && defaultRequireAlias;
                 boolean requireDataStream = defaultRequireDataStream != null && defaultRequireDataStream;
                 Map<String, String> dynamicTemplates = Map.of();
+                Map<String, Map<String, String>> dynamicTemplatesParms = Map.of();
 
                 // at this stage, next token can either be END_OBJECT (and use default index and type, with auto generated id)
                 // or START_OBJECT which will have another set of parameters
@@ -427,19 +429,22 @@ private boolean parseActionLine(BytesReference data, int from, int to) throws IO
                             && DYNAMIC_TEMPLATES.match(currentFieldName, parser.getDeprecationHandler())) {
                                 dynamicTemplates = parser.mapStrings();
                             } else if (token == XContentParser.Token.START_OBJECT
-                                && SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
-                                    currentFetchSourceContext = FetchSourceContext.fromXContent(parser);
-                                } else if (token != XContentParser.Token.VALUE_NULL) {
-                                    throw new IllegalArgumentException(
-                                        "Malformed action/metadata line ["
-                                            + line
-                                            + "], expected a simple value for field ["
-                                            + currentFieldName
-                                            + "] but found ["
-                                            + token
-                                            + "]"
-                                    );
-                                }
+                                && DYNAMIC_TEMPLATES_PARAMS.match(currentFieldName, parser.getDeprecationHandler())) {
+                                    dynamicTemplatesParms = parser.map(HashMap::new, XContentParser::mapStrings);
+                                } else if (token == XContentParser.Token.START_OBJECT
+                                    && SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
+                                        currentFetchSourceContext = FetchSourceContext.fromXContent(parser);
+                                    } else if (token != XContentParser.Token.VALUE_NULL) {
+                                        throw new IllegalArgumentException(
+                                            "Malformed action/metadata line ["
+                                                + line
+                                                + "], expected a simple value for field ["
+                                                + currentFieldName
+                                                + "] but found ["
+                                                + token
+                                                + "]"
+                                        );
+                                    }
                     }
                 } else if (token != XContentParser.Token.END_OBJECT) {
                     throw new IllegalArgumentException(
@@ -462,6 +467,11 @@ private boolean parseActionLine(BytesReference data, int from, int to) throws IO
                             "Delete request in line [" + line + "] does not accept " + DYNAMIC_TEMPLATES.getPreferredName()
                         );
                     }
+                    if (dynamicTemplatesParms.isEmpty() == false) {
+                        throw new IllegalArgumentException(
+                            "Update request in line [" + line + "] does not accept " + DYNAMIC_TEMPLATES_PARAMS.getPreferredName()
+                        );
+                    }
                     currentRequest = new DeleteRequest(index).id(id)
                         .routing(routing)
                         .version(version)
@@ -480,6 +490,7 @@ private boolean parseActionLine(BytesReference data, int from, int to) throws IO
                             .setIfSeqNo(ifSeqNo)
                             .setIfPrimaryTerm(ifPrimaryTerm)
                             .setDynamicTemplates(dynamicTemplates)
+                            .setDynamicTemplatesParams(dynamicTemplatesParms)
                             .setRequireAlias(requireAlias)
                             .setRequireDataStream(requireDataStream)
                             .setListExecutedPipelines(currentListExecutedPipelines)
@@ -508,6 +519,11 @@ private boolean parseActionLine(BytesReference data, int from, int to) throws IO
                                 "Update request in line [" + line + "] does not accept " + DYNAMIC_TEMPLATES.getPreferredName()
                             );
                         }
+                        if (dynamicTemplatesParms.isEmpty() == false) {
+                            throw new IllegalArgumentException(
+                                "Update request in line [" + line + "] does not accept " + DYNAMIC_TEMPLATES_PARAMS.getPreferredName()
+                            );
+                        }
                         UpdateRequest updateRequest = new UpdateRequest().index(index)
                             .id(id)
                             .routing(routing)

server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -433,6 +433,7 @@ static boolean executeBulkItemRequest(
                 request.getContentType(),
                 request.routing(),
                 request.getDynamicTemplates(),
+                request.getDynamicTemplatesParams(),
                 request.getIncludeSourceOnError(),
                 meteringParserDecorator,
                 request.tsid()
@@ -755,6 +756,7 @@ private static Engine.Result performOpOnReplica(
                     indexRequest.getContentType(),
                     indexRequest.routing(),
                     Map.of(),
+                    Map.of(),
                     true,
                     XContentMeteringParserDecorator.NOOP,
                     indexRequest.tsid()

server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java

@@ -206,6 +206,7 @@ private ValidationResult validateMappings(
             request.getContentType(),
             request.routing(),
             request.getDynamicTemplates(),
+            request.getDynamicTemplatesParams(),
             request.getIncludeSourceOnError(),
             XContentMeteringParserDecorator.NOOP,
             request.tsid()

server/src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -79,6 +79,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     private static final TransportVersion PIPELINES_HAVE_RUN_FIELD_ADDED = TransportVersions.V_8_10_X;
     private static final TransportVersion INDEX_REQUEST_INCLUDE_TSID = TransportVersion.fromName("index_request_include_tsid");
     private static final TransportVersion INDEX_SOURCE = TransportVersion.fromName("index_source");
+    static final TransportVersion INGEST_REQUEST_DYNAMIC_TEMPLATES_PARAMS = TransportVersion.fromName(
+        "ingest_request_dynamic_templates_params"
+    );
 
     private static final Supplier<String> ID_GENERATOR = UUIDs::base64UUID;
 
@@ -147,6 +150,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM;
 
     private Map<String, String> dynamicTemplates = Map.of();
+    private Map<String, Map<String, String>> dynamicTemplatesParams = Map.of();
 
     /**
      * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised.
@@ -234,6 +238,10 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio
         if (in.getTransportVersion().supports(INDEX_REQUEST_INCLUDE_TSID)) {
             tsid = in.readBytesRefOrNullIfEmpty();
         }
+
+        if (in.getTransportVersion().supports(INGEST_REQUEST_DYNAMIC_TEMPLATES_PARAMS)) {
+            dynamicTemplatesParams = in.readMap(StreamInput::readString, i -> i.readMap(StreamInput::readString));
+        }
     }
 
     public IndexRequest() {
@@ -823,6 +831,9 @@ private void writeBody(StreamOutput out) throws IOException {
         if (out.getTransportVersion().supports(INDEX_REQUEST_INCLUDE_TSID)) {
             out.writeBytesRef(tsid);
         }
+        if (out.getTransportVersion().supports(INGEST_REQUEST_DYNAMIC_TEMPLATES_PARAMS)) {
+            out.writeMap(dynamicTemplatesParams, StreamOutput::writeString, (o, v) -> o.writeMap(v, StreamOutput::writeString));
+        }
     }
 
     @Override
@@ -982,6 +993,15 @@ public Map<String, String> getDynamicTemplates() {
         return dynamicTemplates;
     }
 
+    public IndexRequest setDynamicTemplatesParams(Map<String, Map<String, String>> dynamicTemplatesParams) {
+        this.dynamicTemplatesParams = dynamicTemplatesParams;
+        return this;
+    }
+
+    public Map<String, Map<String, String>> getDynamicTemplatesParams() {
+        return dynamicTemplatesParams;
+    }
+
     public Object getRawTimestamp() {
         return rawTimestamp;
     }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -922,6 +922,11 @@ public final DynamicTemplate findDynamicTemplate(String fieldName, DynamicTempla
         return null;
     }
 
+    public final Map<String, String> getDynamicTemplateParams(String fieldName) {
+        final String pathAsString = path().pathAsText(fieldName);
+        return sourceToParse.dynamicTemplatesParams().getOrDefault(pathAsString, Map.of());
+    }
+
     // XContentParser that wraps an existing parser positioned on a value,
     // and a field name, and returns a stream that looks like { 'field' : 'value' }
     private static class CopyToParser extends FilterXContentParserWrapper {

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -237,7 +237,7 @@ private static boolean applyMatchingTemplate(
         String dynamicType = dynamicTemplate.isRuntimeMapping() ? matchType.defaultRuntimeMappingType() : matchType.defaultMappingType();
 
         String mappingType = dynamicTemplate.mappingType(dynamicType);
-        Map<String, Object> mapping = dynamicTemplate.mappingForName(name, dynamicType);
+        Map<String, Object> mapping = dynamicTemplate.mappingForName(name, dynamicType, context.getDynamicTemplateParams(name));
         if (dynamicTemplate.isRuntimeMapping()) {
             MappingParserContext parserContext = context.dynamicTemplateParserContext(dateFormatter);
             RuntimeField.Parser parser = parserContext.runtimeFieldParser(mappingType);
@@ -262,7 +262,7 @@ private static Mapper.Builder findTemplateBuilderForObject(DocumentParserContext
         }
         String dynamicType = matchType.defaultMappingType();
         String mappingType = dynamicTemplate.mappingType(dynamicType);
-        Map<String, Object> mapping = dynamicTemplate.mappingForName(name, dynamicType);
+        Map<String, Object> mapping = dynamicTemplate.mappingForName(name, dynamicType, context.getDynamicTemplateParams(name));
         return parseDynamicTemplateMapping(name, mappingType, mapping, null, context);
     }
 

server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java

@@ -26,6 +26,7 @@
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.function.Predicate;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
@@ -35,6 +36,9 @@
 
 public class DynamicTemplate implements ToXContentObject {
 
+    // Pattern to match {{param}} in dynamic template mappings
+    private static final Pattern DYNAMIC_TEMPLATE_PARAM = Pattern.compile("\\{\\{(.*?)}}");
+
     public enum MatchType {
         /**
          * DEFAULT is set when the user did not explicitly set a match_pattern in their dynamic_templates entry.
@@ -257,7 +261,7 @@ public final String toString() {
     }
 
     @SuppressWarnings("unchecked")
-    static DynamicTemplate parse(String name, Map<String, Object> conf) throws MapperParsingException {
+    static DynamicTemplate parse(String name, Map<String, ?> conf) throws MapperParsingException {
         List<String> match = new ArrayList<>(4); // these pattern lists will typically be very small
         List<String> pathMatch = new ArrayList<>(4);
         List<String> unmatch = new ArrayList<>(4);
@@ -268,7 +272,7 @@ static DynamicTemplate parse(String name, Map<String, Object> conf) throws Mappe
         List<String> unmatchMappingType = new ArrayList<>(4);
         String matchPattern = MatchType.DEFAULT.toString();
 
-        for (Map.Entry<String, Object> entry : conf.entrySet()) {
+        for (Map.Entry<String, ?> entry : conf.entrySet()) {
             String propName = entry.getKey();
             if ("match".equals(propName)) {
                 addEntriesToPatternList(match, propName, entry);
@@ -381,7 +385,7 @@ private static boolean matchPatternsAreDefined(final List<?>... matchLists) {
         return Stream.of(matchLists).anyMatch(Predicate.not(List::isEmpty));
     }
 
-    private static void addEntriesToPatternList(List<String> matchList, String propName, Map.Entry<String, Object> entry) {
+    private static void addEntriesToPatternList(List<String> matchList, String propName, Map.Entry<String, ?> entry) {
         if (entry.getValue() instanceof List<?> ls) {
             for (Object o : ls) {
                 if (o instanceof String s) {
@@ -505,7 +509,9 @@ public String mappingType(String dynamicType) {
         String type;
         if (mapping.containsKey("type")) {
             type = mapping.get("type").toString();
+            type = type.replace("{{dynamic_type}}", dynamicType);
             type = type.replace("{dynamic_type}", dynamicType);
+            type = type.replace("{{dynamicType}}", dynamicType);
             type = type.replace("{dynamicType}", dynamicType);
         } else {
             type = dynamicType;
@@ -532,41 +538,68 @@ public boolean isRuntimeMapping() {
     }
 
     public Map<String, Object> mappingForName(String name, String dynamicType) {
-        return processMap(mapping, name, dynamicType);
+        return mappingForName(name, dynamicType, Map.of());
+    }
+
+    public Map<String, Object> mappingForName(String name, String dynamicType, Map<String, String> params) {
+        return processMap(mapping, name, dynamicType, params);
     }
 
-    private static Map<String, Object> processMap(Map<String, Object> map, String name, String dynamicType) {
+    private static Map<String, Object> processMap(Map<String, Object> map, String name, String dynamicType, Map<String, String> params) {
         Map<String, Object> processedMap = new HashMap<>();
         for (Map.Entry<String, Object> entry : map.entrySet()) {
-            String key = entry.getKey()
-                .replace("{name}", name)
-                .replace("{dynamic_type}", dynamicType)
-                .replace("{dynamicType}", dynamicType);
-            processedMap.put(key, extractValue(entry.getValue(), name, dynamicType));
+            String key = processString(entry.getKey(), name, dynamicType, params);
+            processedMap.put(key, extractValue(entry.getValue(), name, dynamicType, params));
         }
         return processedMap;
     }
 
-    private static List<?> processList(List<?> list, String name, String dynamicType) {
+    private static List<?> processList(List<?> list, String name, String dynamicType, Map<String, String> params) {
         List<Object> processedList = new ArrayList<>(list.size());
         for (Object value : list) {
-            processedList.add(extractValue(value, name, dynamicType));
+            processedList.add(extractValue(value, name, dynamicType, params));
         }
         return processedList;
     }
 
     @SuppressWarnings("unchecked")
-    private static Object extractValue(Object value, String name, String dynamicType) {
+    private static Object extractValue(Object value, String name, String dynamicType, Map<String, String> params) {
         if (value instanceof Map) {
-            return processMap((Map<String, Object>) value, name, dynamicType);
+            return processMap((Map<String, Object>) value, name, dynamicType, params);
         } else if (value instanceof List) {
-            return processList((List<?>) value, name, dynamicType);
-        } else if (value instanceof String) {
-            return value.toString().replace("{name}", name).replace("{dynamic_type}", dynamicType).replace("{dynamicType}", dynamicType);
+            return processList((List<?>) value, name, dynamicType, params);
+        } else if (value instanceof String valueString) {
+            valueString = processString(valueString, name, dynamicType, params);
+            return valueString;
         }
         return value;
     }
 
+    private static String processString(String s, String name, String dynamicType, Map<String, String> params) {
+        s = s.replace("{{name}}", name)
+            .replace("{name}", name)
+            .replace("{{dynamic_type}}", dynamicType)
+            .replace("{dynamic_type}", dynamicType)
+            .replace("{{dynamicType}}", dynamicType)
+            .replace("{dynamicType}", dynamicType);
+
+        if (s.contains("{{") == false) {
+            return s;
+        }
+
+        // Handle {{param}} replacements
+        Matcher matcher = DYNAMIC_TEMPLATE_PARAM.matcher(s);
+        StringBuilder sb = new StringBuilder();
+        while (matcher.find()) {
+            String key = matcher.group(1);
+            String replacement = params.getOrDefault(key, "");
+            matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement));
+        }
+        matcher.appendTail(sb);
+
+        return sb.toString();
+    }
+
     String getName() {
         return name;
     }